The Address Relation Toolbox is an extensible, fast, and entirely
run-time solution to redistributing distributed arrays.  ART does not
actually perform communication, instead, it computes the
mapping of data elements stored on a source processor to data elements
stored on a destination processor.   This mapping is induced by the
high level redistribution operation.  The inspector/executor technique
is used to achieve high performance: after the mapping is computed,
it is stored in a compact form and cached for reuse.  These precomputed 
mappings are then used to assemble and disassemble messages, often at
near memory copy bandwidths.

The ART user interface is independent of the kinds of data
distributions, how mappings are computed, and how they are stored and
cached.  In fact, these mechanisms can be orthogonally extended and
the decision of which to use can even be made at run-time.  Currently,
support for HPF regular block-cyclic distributions is implemented and
tested, as well as four different storage schemes.  ART has been built
for DEC Alpha, IBM PowerPC, IBM Power2, Intel Paragon, and Intel
Pentium machines.

Here is an example that shows how ART is used to redistribute a
1024x1024 array of doubles from a blocked row to a blocked column
distribution:

#include "disttool.h"

double a[1024][256];
double b[256][1024];
double buf[256*256];

/* 
 * 
 * B=A
 *    
 * B and A are 1024x1024 double arrays.  B is distributed (*,BLOCK) over
 * four processors and A is distributed (BLOCK,*) over four processors
 */  
main() 
{
   descriptor da, db;
   relationgroup rsend,rrecv;

   /* Initialize ART */
   init_disttool(10,1024*100);

   /* Build the destination descriptor and source descriptor */
   da=make_descriptor(HPF,2,1024,1024,1,1024,256,4);
   db=make_descriptor(HPF,2,1024,256,4,1024,1024,1);

   /* Build relations for sending */
   rsend=make_relation_group(SRC,mynode(),4,db,NULL,da,NULL,NULL);

   /* Build relations for receiving */
   rrecv=make_relation_group(DST,mynode(),4,db,NULL,da,NULL,NULL);

   /* Send messages (this is a really simple schedule) */
   for (proc=0;proc<3;proc++) {
       assemble_message(get_relation(rsend,proc),a,buf,sizeof(double));
       send(i,buf); /* using whatever communication system is available */
   }

   /* Receive messages */
   for (proc=0;proc<3;proc++) {
       recv(i,buf); /* using whatever communication system is available */
       disassemble_message(get_relation(rrecv,proc),b,buf,sizeof(double));
   }
}
   

The following is the header file "disttool.h", which gives some
idea of the additional functionality:

#include "disttypes.h"
#include "disttool_internal.h"

/* Initialization/Deinitialization */
/*
void init_dist();
void init_internals();
void init_cache();
*/
void init_disttool(unsigned numentries, unsigned long space);
void deinit_disttool();


/* Descriptors */
descriptor  make_descriptor(unsigned disttype,...);
error       getinfo_descriptor(descriptor desc, 
			       char       *infostr,
			       long       infolong,
			       void       **returnptr,
			       long       *returnlong);
error       free_descriptor(descriptor desc);




/* Selectors */
selector    make_selector(unsigned disttype,...);
error       getinfo_selector(selector   sel, 
			     char       *infostr,
			     long       infolong,
			     void       **returnptr,
			     long       *returnlong);
error       free_selector(selector sel);



/* Permutations */
permutation make_permutation(unsigned disttype,...);
error       getinfo_permutation(permutation perm, 
				char        *infostr,
				long       infolong,
				void       **returnptr,
				long       *returnlong);
error       free_permutation(permutation perm);



/* Relations */
relation    make_relation(descriptor  destdesc,
			  selector    destsel, 
			  descriptor  srcdesc,
			  selector    srcsel,
			  permutation perm,
			  unsigned    destproc,
			  unsigned    srcproc);
error       getinfo_relation(relation rel, 
			     char     *infostr,
			     long     infolong,
			     void     **returnptr,
			     long     *returnlong);
error       free_relation(relation rel);

error       begin_traversal(relation rel);

error       seek_relation(relation rel, indextype tupleno);

error       next_tuple(relation rel,indextype *sendadx, indextype *recvadx);

error       assemble_part(register relation rel,
			  register indextype numtuples,
			  register char *src,
			  register char *buf,
			  register unsigned typesize);

error       disassemble_part(register relation rel,
			     register indextype numtuples,
			     register char *dst,
			     register char *buf,
			     register unsigned typesize);


     
error       end_traversal(relation rel);
     
error       assemble_message(register relation rel,
			     register char     *src,
			     register char     *buf,
			     register unsigned typesize);
     
error       disassemble_message(relation rel,
				register char     *dst,
				register char     *buf,
				register unsigned typesize);

error       force_final_form(relation rel, finalhandlers *h);

error       make_adpairlist(relation rel,
			    unsigned adpairlist[][2]);
error       make_adblocklist(relation rel,
			     unsigned adblklist[][3]);
error       make_adslicelist(relation rel,
			     unsigned adslclist[][4]);

/* Relation groups */
relationgroup new_relation_group(relationgroupkind kind, unsigned proc);
relationgroup make_relation_group(relationgroupkind kind, 
				  unsigned    thisproc,
				  unsigned    numprocs,
				  descriptor  destdesc,
				  selector    destsel, 
				  descriptor  srcdesc,
				  selector    srcsel,
				  permutation perm);
error         add_relation(relationgroup group, 
			   unsigned proc, 
			   relation rel);
relation      get_relation(relationgroup group, 
			   unsigned proc);
error         delete_relation(relationgroup group, 
			      unsigned proc);
error         getinfo_relationgroup(relationgroup relgrp, 
				    char          *infostr,
				    long          infolong,
				    void          **returnptr,
				    long          *returnlong);
error         free_relation_group(relationgroup group);



/* Cache management */
error        hint_usage(relation rel, cachehint ch);
error       flush_relation(relation rel);
error       lock_relation(relation rel);
error       unlock_relation(relation rel);