///////////////////////////////////////////////////////////////////////////////
//                                                                           //
// Copyright (C) 2006 by Intel Corporation and Carnegie Mellon University    //
// Contacts: casey.j.helfrich @ intel.com                                    //
//           bdr @ cs.cmu.edu                                                //
//                                                                           //
///////////////////////////////////////////////////////////////////////////////

#include "SmartYield.hxx"
#include "GlobalConstants.hxx"

void init_yield(int nthreads, pt_yield *yield) {
  yield->nthreads = nthreads;
  yield->nyielded = 0;
  pthread_mutex_init(&yield->mutex, NULL);
  pthread_cond_init(&yield->condvar[0], NULL);
  pthread_cond_init(&yield->condvar[1], NULL);
  yield->currentcondvar = 0;
  pthread_cond_init(&yield->all_yielded, NULL);
  pthread_cond_init(&yield->resumemaster, NULL);
  pthread_cond_init(&yield->master_yielded, NULL);
  yield->pausemaster = 0;
}

void destroy_yield(pt_yield *yield) {
  yield->nthreads = 0;
  yield->nyielded = 0;
  pthread_mutex_destroy(&yield->mutex);
  pthread_cond_destroy(&yield->condvar[0]);
  pthread_cond_destroy(&yield->condvar[1]);
  yield->currentcondvar = 0;
  pthread_cond_destroy(&yield->all_yielded);
  pthread_cond_destroy(&yield->resumemaster);
  pthread_cond_destroy(&yield->master_yielded);
  yield->pausemaster = 0;
}

void incr_nthreads(pt_yield *yield) {
  pthread_mutex_lock(&yield->mutex);
  yield->nthreads++;
  pthread_mutex_unlock(&yield->mutex);
}

void decr_nthreads(pt_yield *yield) {
  pthread_mutex_lock(&yield->mutex);
  if (--(yield->nthreads) == yield->nyielded) {
    // tell the master to start
    pthread_cond_broadcast(&yield->all_yielded);
  }
  pthread_mutex_unlock(&yield->mutex);
}


/////////////////////
// do_yield
//
// catom thread yield

void do_yield(pt_yield *yield) {
  // because we need to mess with the nyielded variable
  pthread_mutex_lock(&yield->mutex);

  // I am the last thread to yield, need to wake master
  if(++(yield->nyielded) == yield->nthreads) {
    if DPR_THREADING_DEBUG
      cout << yield->nyielded << " of " << yield->nthreads 
	   << " threads yielded..." << endl;

    // tell the master to start
    pthread_cond_broadcast(&yield->all_yielded);
  }
  else {
    if DPR_THREADING_DEBUG
      cout << yield->nyielded << " of " << yield->nthreads 
	   << " threads yielded..." << endl;
  }

  // atomically block the signal (that could be 
  // sent from halt_all and next_time_step) and wait; 
  // since already yielded, the block disables 
  // the handler, and the thread does nothing
  int ret;
  sigset_t set;

  if(sigemptyset(&set) != 0) {
    cerr << "Error: unable to empty signal set :: " << strerror(errno) << endl;
    exit(0);
  }

  // block SIGALRM
  if(sigaddset(&set, SIGALRM) != 0) {
    cerr << "Error: unable to add signal to signal set :: " 
	 << strerror(errno) << endl;
    exit(0);
  }

  if((ret = pthread_sigmask(SIG_BLOCK, &set, NULL)) != 0) {
    cerr << "Error: unable to block signal :: " << strerror(ret) << endl;
    exit(0);
  }

  // now go to sleep on condvar, until woken up
  pthread_cond_wait(&yield->condvar[yield->currentcondvar], &yield->mutex);

  // atomically wakes up (from previous) and unblock the signal
  if((ret = pthread_sigmask(SIG_UNBLOCK, &set, NULL)) != 0) {
    cerr << "Error: unable to unblock signal :: " << strerror(ret) << endl;
    exit(0);
  }

  // need to release, since pthread_cond_wait re-acquires
  pthread_mutex_unlock(&yield->mutex);
}

//////////////////////////
// do_master_yield
//
// master's yield function

void do_master_yield(pt_yield *yield) {
  // go to critical section
  pthread_mutex_lock(&yield->mutex);

  // if halt_all was called -- this 'redundant' check 
  // is to ensure that halt_all does not have to wait 
  // one whole timestep to gain control if it signals 
  // while the master is computing
  if(yield->pausemaster) {
    pthread_cond_broadcast(&yield->master_yielded);

    // wait for next_time_step call
    pthread_cond_wait(&yield->resumemaster, &yield->mutex);
  }

  // wake up threads from last yield
  pthread_cond_broadcast(&yield->condvar[1 - yield->currentcondvar]);

  // make sure all threads haven't yet yielded
  if(yield->nyielded < yield->nthreads) {
    // wait for all threads to yield
    pthread_cond_wait(&yield->all_yielded, &yield->mutex);
  }

  /* all threads will be waiting on condvar[currentcondvar] at this point */

  // if halt_all was called -- presumably halt_all was 
  // called while the master was sleeping (while the 
  // catoms are computing)
  if(yield->pausemaster) {
    pthread_cond_broadcast(&yield->master_yielded);

    // wait for next_time_step call
    pthread_cond_wait(&yield->resumemaster, &yield->mutex);
  }

  yield->currentcondvar = 1 - yield->currentcondvar;
  yield->nyielded = 0;
  pthread_mutex_unlock(&yield->mutex);
}

////////////////////////////////////////////////////////
// halt_all
//
// blocking call that halts all threads and the master; 
// resume using next_time_step
// notice that, although it sends signal to immediately 
// halt catom threads, it respects the master thread 
// to finish its computation first (if halt_all is 
// called while the master thread is coputing)

void halt_all(pt_yield *yield, vector<pthread_t> &catomThreadList) {
  // go to critical section
  pthread_mutex_lock(&yield->mutex);

  // ensure master waits
  yield->pausemaster = 1;

  if(yield->nyielded < yield->nthreads) {
    // signal all threads
    int ret;

    for(unsigned int i = 0; i < catomThreadList.size(); i++) {
      if((ret = pthread_kill(catomThreadList[i], SIGALRM)) != 0) {
	cerr << "Error: unable to send signal :: " << strerror(errno) << endl;
	exit(0);
      }
    }

    // wait for all catom threads and master thread to yield
    pthread_cond_wait(&yield->master_yielded, &yield->mutex);
  }

  // exit critical section
  pthread_mutex_unlock(&yield->mutex);
}

////////////////////////////////////////////////////////
// next_time_step
//
// non-blocking call, triggers all threads to yield and 
// master to progress time; also resumes master after 
// halt_all

void next_time_step(pt_yield *yield, vector<pthread_t> &catomThreadList) {
  // go to critical section
  pthread_mutex_lock(&yield->mutex);

  if(yield->nyielded < yield->nthreads) {
    //yield->nyielded = 0;

    // signal all threads
    int ret;

    for(unsigned int i = 0; i < catomThreadList.size(); i++) {
      if((ret = pthread_kill(catomThreadList[i], SIGALRM)) != 0) {
	cerr << "Error: unable to send signal :: " << strerror(errno) << endl;
	exit(0);
      }
    }
  }

  // release master from pause
  if(yield->pausemaster) {
    yield->pausemaster = 0;
    pthread_cond_broadcast(&yield->resumemaster);
  }

  // exit critical section
  pthread_mutex_unlock(&yield->mutex);
}

///////////////////////////////////////////////////////
// signal_handler
//
// thread signal handler to force yielding; assumes 
// any thread already yielded gets EINTR from cond_wait

void signal_handler(int sig) {
  // yield right away
  extern pt_yield yield;
  do_yield(&yield);
}
