function NYUIter(instanceId)
%try
  globals;
  if nargin < 1
    instanceId = 1;
  end


  global CHUNK_NUM;
  rootDir = [CONFIG.processingDir '/' CONFIG.processingName '_' num2str(CHUNK_NUM) '/'];

  fprintf('\n\nroot dir = %s\n\n',rootDir);

  fprintf('making root dir\n');
  if ~exist(rootDir, 'dir')
    mkdir(rootDir);
  end
  
  dataSet = 'NYU';
   fprintf('Loading %s\n', dataSet);
  pascalData = getDataSet(dataSet, 'train', CONFIG);
  augData = getDataSet('NYUHN','train',CONFIG);
   
  categories = getAllCategories('NYU', CONFIG);
  pascalSplits = getTrainValSplitForCategory(pascalData.data, categories);
  fprintf('Loading negatives\n');
  hayesData = getDataSet('hayes6Million', 'train', CONFIG);

  outDir = [rootDir 'div2/'];
  if ~exist(outDir, 'dir')
    mkdir(outDir);
  end
  imgsHome.pos = pascalData.imgsHome;
  imgsHome.neg = hayesData.imgsHome;
  imgsHome.aug = augData.imgsHome;
  fprintf('About to process categories\n');
  processCategory(categories, pascalSplits.allPos, hayesData.data, augData.data, imgsHome, ...
    outDir, instanceId);
  
  disp('Done processing all categories');
end

function categories = getAllCategories(mode, config)
switch(mode)
  case 'NYU'
    categories = {''};
end
end

function processCategory(category, posData, negData, augData, homeImgs, outDir, ...
  instanceId)
params = getParamsForCategory(category);
params = params(1);
params.category = category;
params.homeImgs = homeImgs;

splitsFile = [outDir 'SPLITS.mat'];
splitsFlag = [outDir 'SPLITS.flag'];
if ~exist(splitsFile, 'file')
  if instanceId == 1
    splits = getTrainValSplitUnsupervisedRGBD(posData, negData, augData);
    save(splitsFile, 'splits');
    saveTrainingData(splits, outDir);
    createFlagFile(splitsFlag);
  end
end
while ~waitTillExists({splitsFlag})
end
load(splitsFile, 'splits');

trainOutDir = [outDir 'train/'];
if ~exist(trainOutDir, 'dir')
  mkdir(trainOutDir);
end
processImages(splits.allPos, splits.allNeg, splits.allAug, splits.trainSetPos, ...
  splits.validSetPos, splits.trainSetNeg, splits.validSetNeg, ...
  splits.trainSetAug, splits.validSetAug,...
  trainOutDir, params, instanceId);

end

function processImages(trainAllPos, trainAllNeg, trainAllAug, trainSetPos, ...
  validSetPos, trainSetNeg, validSetNeg, ...
  trainSetAug, validSetAug, ...
  outDir, params, instanceId)

rootDir = [outDir '0/'];
if ~exist(rootDir, 'dir')
  mkdir(rootDir);
end
targetFile = 'FIG_PATCH_CLUSTERS_REF';
flagFileName = [rootDir targetFile '.flag'];

% File names
posCalcFeaturesFile = [rootDir 'POS_CALC_FEATURES.mat'];
negCalcFeaturesFile = [rootDir 'NEG_CALC_FEATURES.mat'];
clustOrgFile = [rootDir 'FIG_PATCH_CLUSTERS_ORG.mat'];
refClustFile = [rootDir 'FIG_PATCH_CLUSTERS_REF.mat'];

if instanceId == 1
  if ~exist(posCalcFeaturesFile, 'file')
    fprintf('Generating random patches for positive images ... \n');
    [positivePatches, posFeatures, posCorrespInds] = ...
      getRandomPatchesFromPyramid(trainAllPos(trainSetPos), params, ...
      params.homeImgs.pos);
    posCorrespImgs = trainSetPos(posCorrespInds)';
    fprintf('Done\n');
    save(posCalcFeaturesFile, 'positivePatches', 'posFeatures', ...
      'params', 'posCorrespImgs', 'posCorrespInds');
  else
    load(posCalcFeaturesFile, 'positivePatches', 'posFeatures', ...
      'params', 'posCorrespImgs', 'posCorrespInds');
  end

  % Create some negative features
  numNegImgs = min(10, min(length(trainSetNeg), length(validSetNeg)));
  negImgsSet = [trainSetNeg(1 : numNegImgs) validSetNeg(1 : numNegImgs)]';
  if ~exist(negCalcFeaturesFile, 'file')
    fprintf('Generating random patches for negative images ... \n');
    [negativePatches, negFeatures, negCorrespInds] = ...
      getRandomPatchesFromPyramid(trainAllNeg(negImgsSet), params, ...
      params.homeImgs.neg);
    negCorrespImgs = negImgsSet(negCorrespInds);
    fprintf('Done\n');
    save(negCalcFeaturesFile, 'negativePatches', 'negFeatures', ...
      'negCorrespImgs', 'negCorrespInds');
  else
    load(negCalcFeaturesFile, 'negativePatches', 'negFeatures', ...
      'negCorrespImgs', 'negCorrespInds');
  end
  
  if ~exist(clustOrgFile, 'file')
    disp('Clustering ...');
    toCluster = true(1, size(posFeatures, 1));
    params.numPatchClusters = floor(sum(toCluster) / 4);
    
    fprintf('About to cluster based on normals\n');
    [assignedClust, centers] = clusterNormalAndHOG(positivePatches, posFeatures, params); 
    fprintf('Done clustering. Found %d clusters.\n', size(centers, 1));

    save(clustOrgFile, 'assignedClust', 'centers', ...
      'params', 'positivePatches', 'posFeatures', 'posCorrespImgs', ...
      'posCorrespInds');
  else
    load(clustOrgFile, 'assignedClust', 'centers', ...
      'params', 'positivePatches', 'posFeatures', 'posCorrespImgs', ...
      'posCorrespInds');
  end
  
  if ~exist(refClustFile, 'file')
    fprintf('Refining clusters ... \n');
    [goodClusters, refinedClusters, centers] = refineClustersOverlap( ...
      assignedClust, positivePatches, posFeatures, params, posCorrespInds);
    fprintf('Done refining clusters. %d clusters remaining.\n', ...
      length(goodClusters));
    selectedClusters = goodClusters;
    assignedClust = refinedClusters;
    save(refClustFile, 'assignedClust', ...
      'centers', 'params', 'selectedClusters', 'positivePatches', ...
      'posFeatures', 'posCorrespInds');
  else
    load(refClustFile, 'assignedClust', ...
      'centers', 'params', 'selectedClusters', 'positivePatches', ...
      'posFeatures', 'posCorrespInds');
  end
  
  debug.assignedClustVote = assignedClust;
  debug.assignedClustTrain = assignedClust;
  saveInfoForWarpProcessing(rootDir, assignedClust, ...
    centers, selectedClusters, posFeatures, positivePatches, ...
    trainSetPos, trainSetNeg, params, debug);
  createFlagFile(flagFileName);
  doneProcessing(targetFile, rootDir);
end

while ~waitTillExists({flagFileName})
end

load(refClustFile, 'assignedClust', 'centers', 'params', 'selectedClusters', ...
  'posCorrespInds');
posFeatures = loadAndCheck(refClustFile, 'posFeatures');
positivePatches = loadAndCheck(refClustFile, 'positivePatches');

selectedClustersAll = loadAndCheck(refClustFile, 'selectedClusters');
processingBatchSize = 500;
% processingBatchSize = 3;
currInd = 1;
targetInd = length(selectedClustersAll);
batchInd = 1;
while currInd <= targetInd
  endInd = currInd + processingBatchSize - 1;
  if endInd > targetInd
    endInd = targetInd;
  end
  
  fprintf('Processing batch %d : %d\n', currInd, endInd);
  % Batch flag file
  batchFlagFile = sprintf('%sbatch_%d.flag', rootDir, batchInd);
  if ~fileExists(batchFlagFile)
    selectedClusters = selectedClustersAll(currInd:endInd);
    [batchDir, thisIterDir] = initializeBatchDir(outDir, batchInd, ...
      refClustFile, negCalcFeaturesFile, posCalcFeaturesFile, ...
      selectedClusters, instanceId);
    
    debug.assignedClustVote = assignedClust;
    debug.assignedClustTrain = assignedClust;
    
    warpInfoFlagFile = [thisIterDir 'warp-info.flag'];
    if instanceId == 1
      % Save batch data for processing.
      saveInfoForWarpProcessing(thisIterDir, assignedClust, ...
        centers, selectedClusters, posFeatures, positivePatches, ...
        trainSetPos, trainSetNeg, params, debug);
      createFlagFile(warpInfoFlagFile);
    else
      while ~waitTillExists({warpInfoFlagFile})
      end
    end
    
    doTheIterations(refClustFile, negCalcFeaturesFile, instanceId, ...
      thisIterDir, trainAllPos, trainAllNeg, trainAllAug, trainSetPos, validSetPos, ...
      trainSetNeg, validSetNeg, trainSetAug, validSetAug, batchDir);
    
    if instanceId == 1
      createFlagFile(batchFlagFile);
    end
    while ~waitTillExists({batchFlagFile})
    end

  else
    fprintf('Found batch file [%s], skipping batch.\n', batchFlagFile);
  end
  
  batchInd = batchInd + 1;
  currInd = endInd + 1;
end
end

function [batchDir, batchIter0] = initializeBatchDir(rootDir, batchId, ...
  refClustFile, negCalcFeaturesFile, posCalcFeaturesFile, selectedClusters, ...
  instanceId)
batchDir = sprintf('%sbatch-%d/', rootDir, batchId);
batchIter0 = [batchDir '0/'];
if instanceId == 1
  fprintf('Initializing [%s]\n', batchIter0);
  mkdir(batchIter0);
  dataDir = [rootDir '0/'];
  status = unix(['cp ' posCalcFeaturesFile ' ' batchIter0]);
  status = unix(['cp ' negCalcFeaturesFile ' ' batchIter0]);
  status = unix(['cp ' refClustFile ' ' batchIter0]);
  [prefix, fileName, fileExt] = fileparts(refClustFile);
  save([batchIter0 fileName fileExt ], 'selectedClusters', '-append');
end
end

function doTheIterations(refClustFile, ...
  negCalcFeaturesFile, instanceId, thisIterDir, ...
  trainAllPos, trainAllNeg, trainAllAug, ...
  trainSetPos, validSetPos, trainSetNeg, validSetNeg, ...
  trainSetAug, validSetAug, outDir)

globals;

% Actually do the iterations.
load(refClustFile, 'assignedClust', ...
      'centers', 'params', 'positivePatches', ...
      'posFeatures', 'posCorrespInds');
negFeatures = loadAndCheck(negCalcFeaturesFile, 'negFeatures');
negativePatches = loadAndCheck(negCalcFeaturesFile, 'negativePatches');
negCorrespInds = loadAndCheck(negCalcFeaturesFile, 'negCorrespInds');
negCorrespImgs = loadAndCheck(negCalcFeaturesFile, 'negCorrespImgs');

for i = 0 : 2 : CONFIG.iterCut;
  iterFlagFile = [thisIterDir 'prev_iter_complete.flag'];
  nextIterDir = [outDir sprintf('%d/', i + 1)];
  mkdir(nextIterDir);
  if ~exist(iterFlagFile, 'file')
    storeIterationData(instanceId, thisIterDir, negFeatures, ...
      negCorrespInds, negCorrespImgs, negativePatches, ...
      trainAllPos, trainAllNeg, trainAllAug, ...
      trainSetPos, trainSetNeg, trainSetAug, ...
      validSetPos, validSetNeg, validSetAug);
    fprintf('Starting iteration %s\n', thisIterDir);
    iterateTraining(instanceId, thisIterDir, nextIterDir);
  end
  
  % Quit if iterations complete.
  if i == 4
    break;
  end
  
  thisIterDir = nextIterDir;
  nextIterDir = [outDir sprintf('%d/', i + 2)];
  mkdir(nextIterDir);
  iterFlagFile = [thisIterDir 'prev_iter_complete.flag'];
  % Just flip the information.
  if ~exist(iterFlagFile, 'file')
    storeIterationData(instanceId, thisIterDir, negFeatures, ...
      negCorrespInds, negCorrespImgs, negativePatches, ...
      trainAllPos, trainAllNeg, trainAllAug, ...
      validSetPos, validSetNeg, validSetAug, ...
      trainSetPos, trainSetNeg, trainSetAug);
    fprintf('Starting iteration %s\n', thisIterDir);
    iterateTraining(instanceId, thisIterDir, nextIterDir);
  end
  thisIterDir = nextIterDir;
end
end

function storeIterationData(instanceId, iterDir, negFeatures, ...
  negCorrespInds, negCorrespImgs, negativePatches, ...
  trainAllPos, trainAllNeg, trainAllAug, ...
  trainSetPos, trainSetNeg, trainSetAug, ...
  validSetPos, validSetNeg, validSetAug)
flagFile = [iterDir 'TRAINING_DATA.flag'];
if instanceId == 1 && ~exist(flagFile, 'file')
  save([iterDir 'NEG_CALC_FEATURES'], 'negFeatures', 'negativePatches', ...
    'negCorrespInds', 'negCorrespImgs');
  save([iterDir 'TRAINING_DATA'], 'trainAllPos', 'trainAllNeg', ...
    'trainSetPos', 'trainSetNeg', 'validSetPos', 'validSetNeg', ...
    'trainSetAug', 'validSetAug','trainAllAug');
  createFlagFile(flagFile);
end
while ~waitTillExists({flagFile})
end
end

function iterateTraining(instanceId, thisIterDir, nextIterDir)
globals;
load([thisIterDir 'TRAINING_DATA'], 'trainAllPos', 'trainAllNeg', ...
  'trainAllAug', 'trainSetAug', 'validSetAug', ...
  'trainSetPos', 'trainSetNeg', 'validSetPos', 'validSetNeg');
load([thisIterDir 'FIG_PATCH_CLUSTERS_REF'], ...
    'params', 'selectedClusters');
  
  
initDetectorFile = [thisIterDir 'INIT_DETECTOR_VOTE.mat'];
if ~exist(initDetectorFile, 'file')
  firstTrainDir = [thisIterDir 'firstTrainDir/'];
  fprintf('Running first training script.\n');
  warpFirstTrainScript(instanceId, thisIterDir, firstTrainDir);
  clusterInformation = loadAndCheck([thisIterDir 'CLUSTER_PROCESSING_INFO'], ...
    'clusterInformation');
  if instanceId == 1
    detectors = constructInitialDetectors(firstTrainDir, ...
      clusterInformation, [], [], params);
    save([thisIterDir 'INIT_DETECTOR_VOTE'], 'detectors');
  end
end
while ~waitTillExists({initDetectorFile})
end

detectorFile = [thisIterDir 'detectors_MINED.mat'];
if ~exist(detectorFile, 'file')
  hardNegOut = [thisIterDir 'hardNegMineDir/'];
  fprintf('Starting hard mining.\n');
  warpHardNegMineTrainNormal(instanceId, thisIterDir, hardNegOut);
end
while ~waitTillExists({detectorFile})
end

detectionOut = [thisIterDir 'detectionResult/'];
detectionFile = [detectionOut 'all_detections.mat'];
if ~exist(detectionFile, 'file')
  mkdir(detectionOut);
  warpDetectPresenceScript(instanceId, thisIterDir, detectionOut);
end
while ~waitTillExists({detectionFile})
end

flagFile = [thisIterDir 'prev_iter_complete.flag'];
if instanceId == 1 && ~exist(flagFile, 'file')
  fprintf('Finalizing iteration: %s\n', thisIterDir);
  load(detectionFile, 'allDetections');
  detectionResult = PresenceDetectionResults(detectionOut);
  includeNegatives = false;
  numTrain = length(trainSetPos);
  numValid = length(validSetPos);
  
  % Ignore negs as we don't include them.
%   negInds = validSetNeg(1 : length(posInds));
  numTopN = 20;
  maxOverlap = 0.1;
  trainTopN = getTopNDetsPerCluster(detectionResult, maxOverlap, ...
    trainSetPos, numTopN);
  save([thisIterDir 'TRAIN_TOP_N'] , 'trainTopN');
  topN = getTopNDetsPerCluster(detectionResult, maxOverlap, validSetPos, ...
    numTopN);
  save([thisIterDir 'TOP_N'] , 'topN');

  [posFeatures, positivePatches, ...
    posCorrespInds, posCorrespImgs, assignedClustVote, ...
    assignedClustTrain, selectedClusters] = ...
    prepareDetectedPatchClusters(topN, ...
      10, 5, params, validSetPos, selectedClusters);
  posCorrespImgs = validSetPos(posCorrespInds);
  assignedClust = assignedClustTrain;
  centers = calculateClusterCenters(selectedClusters, assignedClust, ...
    posFeatures);
  save([nextIterDir 'FIG_PATCH_CLUSTERS_REF'], 'assignedClust', ...
    'centers', 'params', 'selectedClusters', 'positivePatches', ...
    'posFeatures', 'posCorrespInds');
  save([nextIterDir 'POS_CALC_FEATURES'], 'posFeatures', ...
    'positivePatches', 'posCorrespInds', 'posCorrespImgs', 'params');

  debug.assignedClustVote = assignedClustVote;
  debug.assignedClustTrain = assignedClustTrain;

  saveInfoForWarpProcessing(nextIterDir, assignedClust, ...
    centers, selectedClusters, posFeatures, positivePatches, ...
    validSetPos, validSetNeg, params, debug);
  createFlagFile(flagFile);
end
while ~waitTillExists({flagFile})
end
end

function saveTrainingData(splits, outDir)
trainAllPos = splits.allPos;
trainAllNeg = splits.allNeg;
trainAllAug = splits.allAug;
trainSetPos = splits.trainSetPos;
trainSetNeg = splits.trainSetNeg;
trainSetAug = splits.trainSetAug;
validSetPos = splits.validSetPos;
validSetNeg = splits.validSetNeg;
validSetAug = splits.validSetAug;
save([outDir 'TRAINING_DATA'], 'trainAllPos', 'trainAllNeg', ...
  'trainSetPos', 'trainSetNeg', 'validSetPos', 'validSetNeg',...
  'trainAllAug','trainSetAug','validSetAug');
end
