 \documentstyle[11pt,fullpage,cmu-titlepage,prog]{cmu-art}

\title{iWarp Local Guide}
\author{Michael D. Mills}
\date{September 17, 1992}
\trnumber{}

\citationinfo{
\begin{center}
/afs/cs/project/iwarp/doc/local.ps
\end{center}
The work was supported in part by Defense Advanced Research
Projects Agency (DOD) monitored by DARPA/CMO under Contract
MDA972-90-C-0035, and in part by the Air Force Office of Scientific
Research under grant F49620-92-J-0131.
}

\abstract{
This document explains to new users how to use the iWarp systems at
Carnegie Mellon.  We give an overview of the hardware and software,
explain how to run programs, and explain what to do if you have
problems or questions.  }

\begin{document}
\bibliographystyle{plain}
\maketitle

\section{Introduction}
\label{sec:intro}
This is the iWarp Local Guide. Its purpose is to give new users a
high--level introduction to the iWarp systems at Carnegie Mellon.  It
includes an overview of the hardware and software, a description of
how to compile and run an example program, some hints for debugging
iWarp programs, where to find documentation, and where to go for help.

Because the system is still evolving, be sure you have the most recent
version of this guide, which you can find online in
\begin{prog}
	/afs/cs/project/iwarp/doc/local.ps
\end{prog}

Throughout this guide, there are numerous file references such as the one
above which have the form:
\begin{prog} 
	/afs/cs/project/iwarp/{file} 
\end{prog} 
This is a volume in the Andrew File System (AFS) which is accessible from
most, if not all, networked machines on the Carnegie Mellon campus.  On iWarp
machines, the volume is accessible through this link:
\begin{prog} 
	/iwarp/{file} 
\end{prog}
For example, on iWarp machines, these are equivalent:
\begin{prog} 
	/iwarp/doc/local.ps == /afs/cs/project/iwarp/doc/local.ps
\end{prog}

\subsection{iWarp manual pages}

Use the command {\tt /afs/cs/project/iwarp/bin/iwman} to read iWarp manual
pages.  For example:  
\begin{prog} 
	iwman iwarp 
\end{prog}
will display the manual pages that give a summary of the software provided by
Intel.  A full list of current man pages is available in Appendix A of this
document.

\subsection{The iWarp bboard (newsgroup)}

The {\tt iwarp} bboard is another useful source of information about the
state of the iWarp systems, new software releases, bug fixes, etc. and
is a general way of communicating with iWarp project members.

{\tt Bboard} is a term used within SCS (School of Computer Science) to refer
to local newsgroups.  Whilst the name {\tt iwarp} is recognised as a bboard
on SCS machines, it can also be accessed as a newsgroup across all CMU
machines as {\tt cmu.cs.proj.warp.iwarp}.  

Various bboard/newsgroup reading packages are available at CMU; {\tt bb} and
{\tt rn} are basic readers that are typically available on any Unix machine.

For SCS machines, information about the bboard system and various readers is
available in the SCS general guide.  SCS uses a newer threaded version of rn
called {\tt trn}.  We also have an Emacs and Gnu-Emacs based reader called
{\tt bags}, a subset of the Rand MH mail program called {\tt msh}, and an
X-windows based program called {\tt xrn}.  Manual pages are available for all
of these (i.e. {\tt man bags}).

For Andrew machines, there are a number of readers including {\tt batmail},
{\tt vui} and {\tt messages}.

\section{Hardware overview}
\label{sec:overview}

There are currently two 20 MHz (C-step) 64--cell iWarp systems at CMU:
Victory and Stargazer.  Figure \ref{fig:iwarp} shows the setup for a typical
64--cell system.  The Sun Interface Board (SIB) includes an iWarp cell and a
64 Kbyte dual--ported memory for talking to the Sun VME bus. Each SIB
contains 2 MBytes of 25ns SRAM.  The Intel numbering scheme for a 64--cell
array is shown in the figure.

\paragraph{For more information:}
See \cite{borkar90b,borkar88,borkar90}.

\begin{figure}
\setlength{\unitlength}{0.1in}
\scriptsize
\begin{center}
\begin{picture}(38,34)(0,0)

%sib
\put(2,30){\framebox(2,2){}}
\put(0,31){\line(1,0){2}}

%cells
\multiput(6,30)(4,0){8}{\framebox(2,2){}}
\multiput(6,26)(4,0){8}{\framebox(2,2){}}
\multiput(6,22)(4,0){8}{\framebox(2,2){}}
\multiput(6,18)(4,0){8}{\framebox(2,2){}}
\multiput(6,14)(4,0){8}{\framebox(2,2){}}
\multiput(6,10)(4,0){8}{\framebox(2,2){}}
\multiput(6,6)(4,0){8}{\framebox(2,2){}}
\multiput(6,2)(4,0){8}{\framebox(2,2){}}

%horizontal links
\multiput(4,31)(4,0){9}{\line(1,0){2}}
\multiput(4,27)(4,0){9}{\line(1,0){2}}
\multiput(4,23)(4,0){9}{\line(1,0){2}}
\multiput(4,19)(4,0){9}{\line(1,0){2}}
\multiput(4,15)(4,0){9}{\line(1,0){2}}
\multiput(4,11)(4,0){9}{\line(1,0){2}}
\multiput(4,7)(4,0){9}{\line(1,0){2}}
\multiput(4,3)(4,0){9}{\line(1,0){2}}

%vertical links
\multiput(7,0)(0,4){9}{\line(0,1){2}}
\multiput(11,0)(0,4){9}{\line(0,1){2}}
\multiput(15,0)(0,4){9}{\line(0,1){2}}
\multiput(19,0)(0,4){9}{\line(0,1){2}}
\multiput(23,0)(0,4){9}{\line(0,1){2}}
\multiput(27,0)(0,4){9}{\line(0,1){2}}
\multiput(31,0)(0,4){9}{\line(0,1){2}}
\multiput(35,0)(0,4){9}{\line(0,1){2}}

%cell numbers
%sib
\put(3,31){\makebox(0,0){64}}
\put(1,29){\dashbox{.25}(4,4){}}
\put(3,33.2){\makebox(0,0)[b]{SIB}}

%first row
\put(07,31){\makebox(0,0){0}}
\put(11,31){\makebox(0,0){1}}
\put(15,31){\makebox(0,0){2}}
\put(19,31){\makebox(0,0){3}}
\put(23,31){\makebox(0,0){4}}
\put(27,31){\makebox(0,0){5}}
\put(31,31){\makebox(0,0){6}}
\put(35,31){\makebox(0,0){7}}

%second row
\put(07,27){\makebox(0,0){8}}
\put(11,27){\makebox(0,0){9}}
\put(15,27){\makebox(0,0){10}}
\put(19,27){\makebox(0,0){11}}
\put(23,27){\makebox(0,0){12}}
\put(27,27){\makebox(0,0){13}}
\put(31,27){\makebox(0,0){14}}
\put(35,27){\makebox(0,0){15}}

%second row
\put(07,23){\makebox(0,0){16}}
\put(11,23){\makebox(0,0){17}}
\put(15,23){\makebox(0,0){18}}
\put(19,23){\makebox(0,0){19}}
\put(23,23){\makebox(0,0){20}}
\put(27,23){\makebox(0,0){21}}
\put(31,23){\makebox(0,0){22}}
\put(35,23){\makebox(0,0){23}}

%third row
\put(07,19){\makebox(0,0){24}}
\put(11,19){\makebox(0,0){25}}
\put(15,19){\makebox(0,0){26}}
\put(19,19){\makebox(0,0){27}}
\put(23,19){\makebox(0,0){28}}
\put(27,19){\makebox(0,0){29}}
\put(31,19){\makebox(0,0){30}}
\put(35,19){\makebox(0,0){31}}

%fourth row
\put(07,15){\makebox(0,0){32}}
\put(11,15){\makebox(0,0){33}}
\put(15,15){\makebox(0,0){34}}
\put(19,15){\makebox(0,0){35}}
\put(23,15){\makebox(0,0){36}}
\put(27,15){\makebox(0,0){37}}
\put(31,15){\makebox(0,0){38}}
\put(35,15){\makebox(0,0){39}}

%fifth row
\put(07,11){\makebox(0,0){40}}
\put(11,11){\makebox(0,0){41}}
\put(15,11){\makebox(0,0){42}}
\put(19,11){\makebox(0,0){43}}
\put(23,11){\makebox(0,0){44}}
\put(27,11){\makebox(0,0){45}}

\put(31,11){\makebox(0,0){46}}
\put(35,11){\makebox(0,0){47}}

%sixth row
\put(07,7){\makebox(0,0){48}}
\put(11,7){\makebox(0,0){49}}
\put(15,7){\makebox(0,0){50}}
\put(19,7){\makebox(0,0){51}}
\put(23,7){\makebox(0,0){52}}
\put(27,7){\makebox(0,0){53}}
\put(31,7){\makebox(0,0){54}}
\put(35,7){\makebox(0,0){55}}

%seventh row
\put(07,3){\makebox(0,0){56}}
\put(11,3){\makebox(0,0){57}}
\put(15,3){\makebox(0,0){58}}
\put(19,3){\makebox(0,0){59}}
\put(23,3){\makebox(0,0){60}}
\put(27,3){\makebox(0,0){61}}
\put(31,3){\makebox(0,0){62}}
\put(35,3){\makebox(0,0){63}}

\end{picture}
\end{center}
\caption{iWarp System}
\label{fig:iwarp}
\end{figure}

\section{Software Overview}
\label{sec:software}

\subsection{C Compiler}
The iWarp C compiler ({\tt iwcc}) is a K\&R C compiler with
some extensions that support direct access of the systolic gates and
assembly language inlining.

\paragraph{For more information:}
See \cite{reinders89} and the man page {\tt iwcc(1)}.

\subsection{Fortran compiler}

The iWarp FORTRAN compiler ({\tt iwf77}) is an optimising version of f77
supported and distributed by Intel.  It produces COFF (Common Object File
Format) object code, and recognises VMS extensions and various optimisation
methods including preprocessing and inlining.

\paragraph{For more information:}
See the man page {\tt iwf77(1)}.

\subsection{PCS} Programmed Communication Services (PCS) is a collection of
tools and library routines for writing iWarp programs. Originally developed
at Carnegie Mellon, it is now supported and distributed by Intel.  PCS
provides an array--level view of the iWarp, and efficient and direct program
access to the iWarp's communication hardware.  This section gives a brief
overview of the PCS programming model, and the tools and library routines
that support the model.

\subsubsection{PCS programming model}
PCS programs are based on the notion of a {\em module}.  Modules are
hierarchical entities, powerful abstractions that allow programs to be
constructed from smaller programs. A module is either a {\em cell
program}, which is a single-threaded C {\tt main()} program that runs
on one cell, or a collection of instances of modules and the networks
that connect them. Networks can be simple point-to-point networks, or
more sophisticated multi-source, multi-drop networks.

Modules run on {\em logical arrays} of cells.  The cells in a logical
array are numbered as shown in Figure \ref{fig:logicalarray}. On our
systems at Carnegie Mellon, there is one auxiliary cell $(0,-1)$
which corresponds to the SIB.

\begin{figure}
\setlength{\unitlength}{0.2in}
\scriptsize
\begin{center}
\begin{picture}(20,17)(3,14)

%cells
\multiput(6,30)(4,0){4}{\framebox(2,2){}}
\multiput(6,26)(4,0){4}{\framebox(2,2){}}
\multiput(6,22)(4,0){4}{\framebox(2,2){}}
\multiput(6,18)(4,0){4}{\framebox(2,2){}}
\put(11.75,14){\framebox(2.5,2){}}

%cell numbers
%first row
\put(07,31){\makebox(0,0){$(0,0)$}}
\put(11,31){\makebox(0,0){$(0,1)$}}
\put(15,31){\makebox(0,0){$(0,2)$}}
\put(19,31){\makebox(0,0){$(0,3)$}}

%second row
\put(07,27){\makebox(0,0){$(1,0)$}}
\put(11,27){\makebox(0,0){$(1,1)$}}
\put(15,27){\makebox(0,0){$(1,2)$}}
\put(19,27){\makebox(0,0){$(1,3)$}}

%third row
\put(07,23){\makebox(0,0){$(2,0)$}}
\put(11,23){\makebox(0,0){$(2,1)$}}
\put(15,23){\makebox(0,0){$(2,2)$}}
\put(19,23){\makebox(0,0){$(2,3)$}}

%fourth row
\put(07,19){\makebox(0,0){$(3,0)$}}
\put(11,19){\makebox(0,0){$(3,1)$}}
\put(15,19){\makebox(0,0){$(3,2)$}}
\put(19,19){\makebox(0,0){$(3,3)$}}

%fifth row
\put(13,15){\makebox(0,0){$(0,-1)$}}
\end{picture}
\end{center}
\caption{Numbering scheme for logical arrays}
\label{fig:logicalarray}
\end{figure}

\subsubsection{PCS tools}
{\tt iwcb} is the array--level linker. It inputs a C program called an
{\em array program}, and outputs a single file, called an {\em array
module}, that can be loaded and executed on the iWarp array. The array
program describes in a textual way instances of modules and the
networks that connect them.  Modules and networks can be placed and
routed manually, by the user, or automatically, by {\tt iwcb}.

{\tt iwamld} is the array--level loader.  It inputs the array module produced
by {\tt iwcb} and loads and executes it on the iWarp array.  The {\tt iwamld}
program is smart enough to produce only one load image for cells that run the
same cell program.  Since each distinct load image requires on the order of
1-2 minutes to compile and link, there is a strong motivation during code
development to write array programs that consist of instances of only a few
different types of cell programs.

Other PCS tools include {\tt iwmd}, a handy X program that graphically
displays the array modules produced by {\tt iwcb}, and {\tt iwpp}, a
preprocessor that inlines function calls to the PCS library.

\subsubsection{PCS library}
PCS includes an extensive library of communications routines for
accessing the networks defined by the array program. Examples include
routines for transferring single words, blocks of data, and entire
messages via the spooling gates or the systolic gates.

\paragraph{For more information:}
See \cite{shinrich91} for details on PCS. See also {\tt
/afs/cs/project/iwarp/examples} for numerous examples of PCS programs.

\subsection{RTS 2.9.3}
Version 2.9.3 of RTS (Run Time System) is a simple runtime system that
supports {\tt malloc()} and Unix file I/O from any cell, asynchronous
message passing between arbitrary cells on the array, and asynchronous
message passing at up to 3 MB/s between arbitrary cells and the Sun
host. An RPC-based Unix command interface is provided for basic
functions such as locking and unlocking the array, loading code on a
cell, running code on a cell, peeking and poking cell memory locations
and registers, and querying completion status.  These commands can be
run from any SunOS workstation on the network.

\paragraph{For more information:}
See the man pages for {\tt iwintro(1)}, {\tt iwmon(1)}, {\tt iwrts(1)},
{\tt intro(2)}, {\tt imsg\_send(2)}, and {\tt imsg\_recv(2)}.

\section{Running iWarp Programs}
\label{sec:account}

\subsection{Finding a Sun Workstation}
To use iWarp locally, you will need an account on the iWarp Suns.  The iWarp
software is installed on Sun-4 workstations running SunOs 4.1.1.  A list of
these machines can be found online in
\begin{prog}
	/afs/cs/project/iwarp/machines.
\end{prog}
Best bets are the public Sun-4/330's: {\tt tokay}, {\tt chenin}, {\tt pinot},
and {\tt gamay}.  Most of the lab machines have 24MB of memory, however {\tt
pinot} and {\tt gamay} each have 40MB making them quite attractive for large
compilations.  Please don't run compilations on the iWarp hosts {\tt victory}
and {\tt stargazer}.

\subsection{Setting up your Environment}
\label{sec:env}
In {\tt .login}:
\begin{prog}
	setpath -i1 /afs/cs/project/iwarp/2.9.3
	setpath -i2 /afs/cs/project/iwarp/misc
\end{prog}
Before running on Stargazer, set the following environment variables:
\begin{prog}
	setenv IWARP_HOSTNAME stargazer
	setenv IWARP_SIBNUM 2
	setenv IWARP_CFGPATH /afs/cs/project/iwarp/etc2.9.3/cfg
\end{prog}
Before running on Victory, set the following environment variables:
\begin{prog}
	setenv IWARP_HOSTNAME victory
	setenv IWARP_SIBNUM 0
	setenv IWARP_CFGPATH /afs/cs/project/iwarp/etc2.9.3/cfg
\end{prog}

\subsection{Compiling and running PCS programs}
\label{sec:compiling}

\subsubsection{An example PCS program}
Here is a simple PCS program that sends a message from from one cell to
another. The program illustrates some basic things like how to synchronize
the clocks on a collection of cells, how to write to files, how to transfer
data between cells, how to blink the front panel lights, how to measure
execution time, and how to use the C math library. Sources are in
\begin{prog}
	/afs/cs/project/iwarp/misc/examples/local_guide/{array.c,cell.c,makefile}
\end{prog}
The array program ({\tt array.c}) looks like this:
\begin{prog}
	#include <stdio.h>
	#include <pcs/cb_def.h>

	user_main()
	{
		int i;
		cellid_type src, dst;
		port_type *out_port, *in_port;
		network_type *nw;
  
		src.row = 0; src.col = 0; src.type = ARRAY;
		dst.row = 1; dst.col = 1; dst.type = ARRAY;

		out_port = create_output_port("out", src);
		in_port = create_input_port("in", dst);
		nw = create_network(out_port, in_port, NULL);

		place(src, "cell.o");
		place(dst, "cell.o");
	}
\end{prog}

This array program defines a $2 \times 2$ logical array, with a
point-to-point network connecting cell $(0,0)$ to cell $(1,1)$.  Cells
$(0,1)$ and $(1,0)$ are undefined and inactive.  Connections are created
directionally from one cell ({\tt src}) to another ({\tt dst}).  Cell types
are either {\tt AUX} for the SIB or {\tt ARRAY} for all others.  The {\tt
place} function directs what COFF files are loaded on the specified cell.  It
can be called multiple times to place additional object files on the same
cell.  In this example, the same object file {\tt cell.o} is placed on both
active cells.

The cell program ({\tt cell.c}) that runs on cells $(0,0)$ and $(1,1)$ is:

\begin{prog}	
	#include <stdio.h>
	#include <math.h>
	#include <pcs/pcs_def.h>
	#include <pcs/pcs_time.h>
	#include <ksupp/blink.h>
	#include <iwsys/getcfg.h>

	#define SIZE 1024

	float data[SIZE];

	extern int getcfg();
	struct iwcfg cfg;

	port_type *out, *in;
	port_list_type ports[] = {
		{"out", &out},
		{"in", &in}};

	main(argc, argv)
	int argc;
	char *argv[];
	{

		int i, stime, clocks, bytes, mhz;
		cellid_type *cellid;

		/* create the network */
		pcs_init(ports, NUM_PORTS(ports), NULL, 0);
  
		/* initialize the barrier package */
		getcfg(&cfg);
		sb_init(cfg.cellid);

		/* initialize the data */
		cellid = get_cellid();
		if (cellid->col == 0) 
			for (i=0; i<SIZE; i++) 
				data[i] = (float)sqrt((float)i);

		/* transfer the message */
		ledctl(LED_YELLOW_ON);
		sb_synch(100,2,0);
		stime = GetTimer1();
		EnableTimer1();
		if (cellid->col == 0) {
			bind_systolic_gate(out, GATE0);
			send_direct_msg(out, GATE0, data, SIZE*sizeof(float), NULL);
		}
		else {
			bind_systolic_gate(in, GATE0);
			receive_limited_msg(in, GATE0, data, SIZE*sizeof(float));
		}
		DisableTimer1();
		sb_synch(200,2,1);
		ledctl(LED_YELLOW_OFF);
  
		/* print the performance */
		clocks = (stime-GetTimer1())*8;
		bytes = SIZE*sizeof(float);
		mhz = cfg.syshz/1000000;
		printf("%d bytes\n", bytes);
		printf("%d clocks\n", clocks);
		printf("%.6f seconds\n", (float)clocks/(float)cfg.syshz);
		printf("%.2f clocks/byte\n", (float)clocks/(float)bytes);
		printf("%.2f mbytes/second\n", ((float)bytes/(float)clocks)*(float)mhz);

		/* print the results */
		if (cellid->col) {
			printf("\n");
			for (i=0; i<8; i++)
				printf("%f\n", data[i]);  
		}

		/* close the network */
		pcs_de_init();

		exit(0);
	}
\end{prog}

\subsubsection*{Notes}

The {\tt getcfg} function retrieves configuration information about the iWarp
array and the local cell, and places it into a structure.  Cell position
({\tt row}, {\tt col}), logical number ({\tt cellid}) and clock speed ({\tt
syshz}) are all member variables of this structure.

Barrier synchronisation, which is explained further in the {\tt misc} section
of this document, is used to coordinate the execution of the cells at various
points.  It is initialised with the {\tt sb\_init} function, and invoked with
the {\tt sb\_synch} function.  The cells are synchronised at two points in
this program, before and after the primary action of the program.

The yellow LED is used to give some low-expense visual feedback about cell
activity in this program.  More information on the LEDs and how to view them
is given later in this document.  The function used here is {\tt ledctl}
which is loaded from the {\tt ksupp/blink.h} include file.  That file shows
the various arguments to {\tt ledctl} which are self-explanatory.

The timers in this program are a feature of PCS; calls to {\tt GetTimer1}
return integer values which count in descending order when enabled.  Each
decrement of the Timer value corresponds to approximately 8 (eight) clock
cycles.  In this case, timer values from the beginning and end of the primary
computational segment are compared to calculate the overall elapsed time.

\subsubsection{Compiling and Executing the Example Program}

To compile and run this program on Victory, login to a Sun-4 that runs
SunOS 4.1.1 and invoke the following makefile.  Before you invoke the
makefile, make sure you are authenticated to AFS on Victory by using
the {\tt kinit} program, and that the following environment variables
are defined
\begin{prog}
	setenv IWARP_HOSTNAME victory
	setenv IWARP_SIBNUM 0
	setenv IWARP_CFGPATH /afs/cs/project/iwarp/etc2.9.3/cfg

	CC = iwcc
	CFLAGS = -O -fsingle 
	TARGET_ARCH = iwarp
	LIB = -L/afs/cs/project/iwarp/misc/barrier/2.9.3

	all: array
		iwamld array -W1,$(LIB),-lm,-lsynch

	array: array.c cell.o
		iwcb -row 2 -col 2 -o array array.c

	clean:
		rm -f *.o cb1 array-* array_* array go std[eo]* *~ 
\end{prog}

When you invoke this makefile, the following things happen.  First,
the C compiler inputs {\tt cell.c} and outputs the cell program {\tt
cell.o}. Next, {\tt iwcb} inputs the array program {\tt array.c} and
outputs the array module {\tt array}, which is then loaded and
executed on Victory by {\tt iwamld}.  

In general, the standard output from physical cell $xxxx$ can be found in
\begin{prog}
	./stdoutxxxx
\end{prog}
Similarly for standard input, the input for physical cell $xxxx$ is in  
\begin{prog}
	./stdinxxxx
\end{prog}
and the output from physical cell $xxxx$ to standard error is in
\begin{prog}
	./stderrxxxx
\end{prog}
where xxxx is a 4-digit decimal number with leading zeros.  By
default, {\tt iwamld} loads the logical array into the top left
hand corner of the physical array.  So in the example program logical
cell $(0,0)$ is loaded onto physical cell 0 and logical cell $(1,1)$
is loaded onto physical cell 9.  When the program executes, the output
appears in files {\tt stdout0000} and {\tt stdout0009}.  The physical cell
numbers correspond to a 64-cell iWarp array as used in the example; the
numbers would differ for other array sizes.

\paragraph{For more information:}
See \cite{shinrich91}.

\subsection{The iWarp misc collection}

The example program makes use of several iWarp packages which are
found in the iWarp misc directory:
\begin{prog}
	/afs/cs/project/iwarp/misc
\end{prog}
This area contains collections of useful tools and packages that are
locally maintained.  The first two are employed in the example program.

If not already done, you can update your environment with:
\begin{prog}
	setpath -i1 /afs/cs/project/iwarp/misc
\end{prog}
to create paths to these various collections.

\subsubsection{Barrier Synchronisation}

The synch library gives the programmer control over the execution times
of groups of cells by setting "roadblocks" for each cell which are not
passed until all cells in the group are at that point.

In the example program, we use the function {\tt sb\_synch} to start the
timing of the two cells together.  This sort of usage is especially helpful
in larger arrays where the disparity in time can be significant, and can make
accurate timing comparisons otherwise impossible.  The exact usage of {\tt
sb\_synch}, {\tt sb\_init}, and other functions is explained in the {\tt
synch(3)} man page.

In the makefile for the example, notice the {\tt LIB} variable which includes
the barrier collection, and the {\tt -lsynch} flag in its unorthodox format
to {\tt iwamld} which loads the physical library {\tt
/afs/cs/project/iwarp/misc/barrier/2.9.3/lib/libsynch.a}.

The barrier synchronisation package is in :
\begin{prog}
	/afs/cs/project/iwarp/misc/bin/barrier
\end{prog}
and includes documentation and examples. 

\subsubsection{LED routines (LED controlling)}

The example program makes use of the status lights on the iWarp to give
feedback about program execution.  Each cell on the iWarp has three status
lights {red,yellow,green} which can be controlled by the program executing
on that cell.  The {\tt ledctl(2)} man page gives details on the standard
method of blinking these LEDs.  A more robust package of routines is found
in:
\begin{prog}
	/afs/cs/project/iwarp/misc/led
\end{prog}
in the include file {\tt led.h}.  The {\tt README} file gives an explanation
of the various functions.

\subsubsection{iwspy (LED watching)}

{\tt iwspy} is an X-windows based tool that displays the monitor lights for
an iWarp array and is an enormously helpful debugging aid. {\tt iwspy} can
be run (architecture-independent) as:
\begin{prog}
	/afs/cs/project/iwarp/misc/bin/iwspy
\end{prog}
Usage is
\begin{prog}
	iwspy [ system ].[ sib ] &
\end{prog}
where {\tt system} is the name of the iWarp array you are using and {\tt sib}
is the number of the SIB you are utilising ("iwspy {\tt stargazer.2} \&" for
example).

{\tt iwspy} is also available for use in Gnu-Emacs providing an ascii 
representation of the status lights in a separate window.  The lisp load
file is found in:
\begin{prog}
	/afs/cs/project/iwarp/misc/iwspy/lisp/iwspy.el
\end{prog}

\subsubsection{calctime}

{\tt calctime} is a tool that estimates numbers of execution cycles of a
cell program based on profile data and object codes.  When using 
{\tt calctime}, the object code gathering profile data and the object
code being used in cycle calculation are different.  Thus, the tool can
eliminate from its timings the effects of profiling and/or workarounds.
In addition to being more accurate than the standard profiler (iwprof), 
{\tt calctime} also provides more detailed statistical information.  it
can be found in:
\begin{prog}
	/afs/cs/project/iwarp/misc/calctime
\end{prog}
The {\tt README} file in that directory gives details whilst the 
{\tt examples} directory contains a sample profiling session.

\subsubsection{espl}

{\tt espl} (Efficient Spooling Pathway Library) as a package of inserts
(written in assembly) that provides very fast communication between cells.
It is bundled as an include file for iWarp C programs in:
\begin{prog}
	/afs/cs/project/iwarp/misc/include/espl.h
\end{prog}
See the {\tt espl(3)} man page for details.

\subsubsection{Kernels}

This is a collection of signal-processing and matrix routines.  written
in assembly for inclusion in iWarp C programs, their primary use is for
benchmarking the compiler but can also be useful for applications.  the
directory is:
\begin{prog}
	/afs/cs/project/iwarp/misc/kernels
\end{prog}
and the {\tt README} file is quite informative.


\subsection{Debugging iWarp Programs}
\label{sec:debugging}
A symbolic debugger for the iWarp is now available in:
\begin{prog}
	/afs/cs/project/iwarp/bin/iwdb
\end{prog}
{\tt iwdb} is an extension of the familiar Gnu debugger (gdb).  It 
provides much of the functionality of gdb with added control management
for cells and cell groupings.  The {\tt iwdb(1)} man page has details.

For the more adventurous, {\tt RTS} provides some low--level 
debugging facilities for peeking and poking memory and register
locations. See the {\tt iwmon(1)} man page for details.

The {\tt LED} routines and {\tt iwspy} packages mentioned previously
are very helpful debugging tools.

Be warned that although iWarp has memory protection hardware, it is
not used by the current version of RTS. So it is very easy to
overwrite the monitor or stack and crash the system.

Some catastrophic failures will leave a cell spinning in an event
handler, with no way to use {\tt RTS} to probe the machine.  These
kinds of failures light up the red LED on the front panel.  If your
program turns on the red LED, see Peter Lieu (pjl@cs.cmu.edu), who can
show you techniques for finding out what caused the failure.

Another problem to be aware of during debugging is that if the monitor
crashes on any cell (a common occurrence because of the lack of memory
protection in the current version of RTS), then you may not necessarily
see all of the printf's issues by the cell programs.

User programs are loaded at address $72$ Kbytes, leaving $440$ Kbytes
for program, static and dynamic data, and stacks.  A C ``hello,
world'' program requires roughly $30$ Kbytes.  Use {\tt iwsize} to
check the size of an executable file.

\paragraph{For more information:}
See \cite{miller90} for details on the iWarp assembly language. Also
see the man pages for {\tt iwmon(1)} and {\tt iwsize(1)}.

\section{Where to Find Documentation}
\label{sec:doc}
CMU iWarp documentation and papers can be found online in 
\begin{prog}
	/afs/cs/project/iwarp/doc/
\end{prog}
See the README file for details.  Online postscript versions of
some of Intel's documentation can be found in 
\begin{prog}
	/afs/cs/project/iwarp/doc-intel/
\end{prog}
Release notes for version xxx of Intel's software are in
\begin{prog}
	/afs/cs/project/iwarp/xxx/docs/
\end{prog}
Barbara Grandillo (bag@cs) has hard copies of the Intel iWarp documents in
her office (WeH 8212).

\section{Reporting Problems and Getting Help}
\label{sec:problems}
Peter Lieu (pjl@cs) is the chief point of contact between Carnegie
Mellon iWarp users and Intel. If you need help, or have found
a bug, send mail to Peter. 

\section{Acknowledgements}
Thanks to Dave O'Hallaron, Mike Gillinov, Susan Hinrichs, Jim Stichnoth, 
and Jon Webb for their comments and suggestions.

\newpage
\appendix
\section{iWarp manual pages}
\begin{tabbing}
abcdefghijklmnopqrstuvwxyzabcdefg\= \kill
IWAMLD (1)              \>- PCS array loader \\
IWCB (1)		\>- PCS array combiner \\
IWMD (1)		\>- PCS array module display \\
IWPP (1)		\>- PCS preprocessor \\
PCS\_INTRO (1)		\>- Introduction to the iWarp Programmed Communication Services \\
iwprof (1)		\>- display profile data\\
cell sets (3)		\>- Create sets of cells to load/run on the iWarp system\\
diwas (1)		\>- iWarp disassembler\\
espl (3)		\>- Efficient Spooling Pathway Library\\
imsg\_block (3)		\>- block until input message available\\
imsg\_info (3)		\>- return information about first message on input queue\\
imsg\_open (3)		\>- open HOST device used for iW/RTS message communication\\
imsg\_recv (3)		\>- asynchronously receive a message\\
imsg\_send (3)		\>- asynchronously send a message\\
imsg\_stat (3)		\>- return information about a pending send or receive\\
isib (4)		\>- driver for Intel iWarp Sun-Interface Board\\
isibio (4)		\>- Intel iWarp SIB ioctl operations\\
iwar (1V)		\>- archive and library maintainer for portable archives\\
iwarp (1)		\>- iWarp Software Summary\\
iwas (1)		\>- iWarp assembler\\
iwbrklock (1)		\>- break another owner's lock on an iWarp array.\\
iwbrklock, iwclrstat, iwdepcsr, iwdepmem, iwexcsr, iwexmem, iwhpi, iwkill, iwload, iwlock, iwreset, iwrun, \\
iwstat, iwtime, iwunlock, iwvers, lgo\>- Unix command interface to the iWarp monitor daemon\\
iwcc (1)		\>- iWarp optimizing C compiler\\
iwccomq (1)		\>- iWarp global optimizer\\
iwcfg (5)		\>- iWarp array configuration file\\
iwcfg, cfg\_getsystem, cfg\_readcfg, cfg\_free, cfg\_packcfg, cfg\_selectcell, cfg\_getinfo, \\
cfg\_getattr, cfg\_getmemmap \>- host-side iWarp configuration information interface\\
iwclrstat (1)		\>- clear cell exit status buffer\\
iwconf (1)		\>- the iWarp interactive diagnostic monitor\\
iwcpp (1)		\>- the C language preprocessor\\
iwdb (1)		\>- The iWarp symbolic debugger\\
iwdepcsr (1)		\>- deposit to control-space register on an iWarp cell\\
iwdepmem (1)		\>- deposit data to memory on an iWarp cell\\
iwdump (1)		\>- dump selected parts of an object file\\
iwexcsr (1)		\>- examine control-space register in an iWarp cell\\
iwexmem (1)		\>- examine memory on an iWarp cell\\
iwf77 (1)		\>- iWarp FORTRAN 77 compiler (with VMS extensions)\\
iwfile (1)		\>- determine the type of a file by examining its contents\\
iwhpi (1)		\>- start iW/RTS Host Proxy Interface server\\
iwkill (1)		\>- send a kill signal to a cell\\
iwld (1)		\>- link editor\\
iwliw (1)		\>- Optimizing code generator for iWarp\\
iwload (1)		\>- load iWarp executable object files on cells.\\
iwlock (1)		\>- gain exclusive access to an iWarp array.\\
iwman (1)		\>- display reference manual pages specific to iWarp\\
iwmond (8)		\>- iWarp Monitor daemon\\
iwnm (1)		\>- print name list\\
iwreset (1)		\>- restart iwrts\\
iwrts (7)		\>- The iWarp Run-Time System\\
iwrun (1)		\>- execute iWarp cell programs\\
iwsize (1)		\>- print section sizes in bytes of common object files\\
iwstat (1)		\>- retreive iWarp array status information\\
iwstrip (1)		\>- remove symbols and relocation bits from an object file\\
iwtime (1)		\>- report execution time of an iWarp cell program\\
iwunlock (1)		\>- remove a lock on an iWarp array\\
iwvers (1)		\>- report version of iwmon server\\
lgo (1)			\>- simplified interface to iwmon\\
mdl\_loadargs (3)	\>- Download argc, argv, and envp onto an iWarp array\\
mdl\_loadcoff (3)	\>- Download a COFF file onto an iWarp array\\
synch (3BS)		\>- iWarp barrier synchronization library (libsynch)\\
\\
\end{tabbing}
\newpage
\bibliography{/usr/droh/bib/defs,/usr/droh/bib/refs,/usr/droh/bib/me}
\end{document}
