From 984e9d5cb2f0ce44f853782b1e9b0a09b9e6d8f3 Mon Sep 17 00:00:00 2001 From: Thilina Rathnayake Date: Wed, 16 Dec 2020 01:57:28 -0600 Subject: [PATCH] Import latest changes (#26) * Add a check to find disconnected components * Add face-check consistency * Refactor metrics, build system and implement Grammian * Fix io and example/partition * Fix MG preconditioner and add Grammian --- Makefile | 43 +- examples/gencon.c | 8 +- examples/partition.c | 146 ++++--- src/gencon/connectivity_io.c | 245 +++++++++++ src/gencon/face-check.c | 245 +++++++++++ src/gencon/gencon-impl.h | 12 + src/gencon/gencon.h | 17 +- src/gencon/geometry_io.c | 248 +++++++++++ src/gencon/io.c | 509 ----------------------- src/gencon/mesh.c | 48 ++- src/gencon/segments.c | 31 +- src/genmap-binsort.c | 207 --------- src/genmap-chelpers.c | 28 +- src/genmap-comm.c | 44 +- src/genmap-components.c | 52 +++ src/genmap-eigen.c | 86 +++- src/genmap-fiedler.c | 152 ++++++- src/genmap-handle.c | 31 +- src/genmap-impl.h | 183 ++++---- src/genmap-lanczos.c | 27 +- src/genmap-laplacian-weighted.c | 48 +-- src/genmap-laplacian.c | 38 +- src/genmap-load-balance.c | 70 ++++ src/genmap-matrix-inverse.c | 40 ++ src/genmap-rcb.c | 194 +++++---- src/genmap-rsb.c | 133 +++--- src/genmap-statistics.c | 67 +-- src/genmap.c | 30 +- src/genmap.h | 91 ++-- src/parCon.c | 24 +- src/parRCB.c | 146 ++++--- src/parRSB.c | 234 ++++++----- src/parRSB.h | 10 +- src/precond/genmap-multigrid-csr.c | 6 +- src/precond/genmap-multigrid-flexcg.c | 12 +- src/precond/genmap-multigrid-fmg.c | 5 +- src/precond/genmap-multigrid-precon.h | 15 +- src/precond/genmap-multigrid-projectpf.c | 57 +-- src/precond/genmap-multigrid-rqi.c | 135 ++++-- src/precond/genmap-multigrid-vcycle.c | 2 +- src/sort/binsort.c | 2 - src/sort/hypercube.c | 2 - src/sort/sort-impl.h | 3 +- src/sort/sort.c | 9 +- tests/t200-laplacian.c | 6 +- tests/t210-levels.c | 6 +- tests/t220-interpolation.c | 6 +- tests/t230-flex-cg.c | 16 +- tests/t235-project-pf.c | 16 +- tests/t240-rqi.c | 16 +- 50 files changed, 2226 insertions(+), 1575 deletions(-) create mode 100644 src/gencon/connectivity_io.c create mode 100644 src/gencon/face-check.c create mode 100644 src/gencon/geometry_io.c delete mode 100644 src/gencon/io.c delete mode 100644 src/genmap-binsort.c create mode 100644 src/genmap-components.c create mode 100644 src/genmap-load-balance.c create mode 100644 src/genmap-matrix-inverse.c diff --git a/Makefile b/Makefile index febf9547..e04b4466 100644 --- a/Makefile +++ b/Makefile @@ -1,17 +1,23 @@ +## General build parameters ## DEBUG ?= 0 -PAUL ?= 1 +MPI ?= 1 CC ?= mpicc -CFLAGS ?= -O2 -## ALGO=0 (Lanczos),1 (RQI),2 (FMG) +CFLAGS ?= -g -O0 +BLAS ?= 0 +UNDERSCORE ?= 0 + +## Genmap algorithmic parameters ## +# ALGO = 0 (Lanczos),1 (RQI),2 (FMG) ALGO ?= 0 -MPI ?= 1 +RCB_PRE_STEP ?= 1 +PAUL ?= 1 +GRAMMIAN ?= 0 +## Don't touch what follows ## MKFILEPATH =$(abspath $(lastword $(MAKEFILE_LIST))) SRCROOT_ ?=$(patsubst %/,%,$(dir $(MKFILEPATH))) SRCROOT =$(realpath $(SRCROOT_)) -GSLIBDIR=$(GSLIBPATH) - SRCDIR =$(SRCROOT)/src SORTDIR =$(SRCROOT)/src/sort PRECONDDIR=$(SRCROOT)/src/precond @@ -23,8 +29,9 @@ TESTDIR =$(SRCROOT)/tests TARGET=parRSB LIB=$(BUILDDIR)/lib/lib$(TARGET).a -INCFLAGS=-I$(SRCDIR) -I$(SORTDIR) -I$(PRECONDDIR) -I$(GENCONDIR) -I$(GSLIBDIR)/include -LDFLAGS:=-L$(BUILDDIR)/lib -l$(TARGET) -L $(GSLIBDIR)/lib -lgs -lm +INCFLAGS=-I$(SRCDIR) -I$(SORTDIR) -I$(PRECONDDIR) -I$(GENCONDIR) \ + -I$(GSLIBPATH)/include +LDFLAGS:=-L$(BUILDDIR)/lib -l$(TARGET) -L$(GSLIBPATH)/lib -lgs -lm SRCS =$(wildcard $(SRCDIR)/*.c) SORTSRCS =$(wildcard $(SORTDIR)/*.c) @@ -52,10 +59,27 @@ else ifeq ($(ALGO),1) PP += -DGENMAP_RQI endif +ifneq ($(RCB_PRE_STEP),0) + PP += -DGENMAP_RCB_PRE_STEP +endif + ifneq ($(PAUL),0) PP += -DGENMAP_PAUL endif +ifneq ($(GRAMMIAN),0) + PP += -DGENMAP_GRAMMIAN +endif + +ifneq ($(UNDERSCORE),0) + PP += -DGENMAP_UNDERSCORE +endif + +ifneq ($(BLAS),0) + PP += -DGENMAP_BLAS + LDFLAGS += -L$(BLASLIBPATH) -lblasLapack +endif + ifneq ($(MPI),0) PP += -DMPI endif @@ -77,7 +101,8 @@ ifneq ($(INSTALLDIR),) @mkdir -p $(INSTALLDIR)/lib 2>/dev/null @cp -v $(LIB) $(INSTALLDIR)/lib 2>/dev/null @mkdir -p $(INSTALLDIR)/include 2>/dev/null - @cp $(SRCDIR)/*.h $(SORTDIR)/*.h $(PRECONDDIR)/*.h $(INSTALLDIR)/include 2>/dev/null + @cp $(SRCDIR)/*.h $(SORTDIR)/*.h $(PRECONDDIR)/*.h \ + $(INSTALLDIR)/include 2>/dev/null endif .PHONY: lib diff --git a/examples/gencon.c b/examples/gencon.c index d99f0dde..30036238 100644 --- a/examples/gencon.c +++ b/examples/gencon.c @@ -15,13 +15,13 @@ int main(int argc,char *argv[]){ } Mesh mesh; - read_re2_mesh(&mesh,argv[1],&comm); + read_geometry(&mesh,argv[1],&comm); findMinNeighborDistance(mesh); double tol=(argc>2)?atof(argv[2]):0.2; - findSegments(mesh,&comm,tol); + findSegments(mesh,&comm,tol,0); setGlobalID(mesh,&comm); sendBack(mesh,&comm); matchPeriodicFaces(mesh,&comm); @@ -30,9 +30,9 @@ int main(int argc,char *argv[]){ int len=strlen(co2FileName); assert(len>4 && len #include -#include "conReader.h" +#include +#include +#include -#include "gslib.h" -#include "parRSB.h" #include "quality.h" #define MAXNV 8 /* maximum number of vertices per element */ typedef struct { int proc; - long long id; long long vtx[MAXNV]; } elm_data; +#define EXIT_ERROR() do{\ + MPI_Finalize();\ + return EXIT_FAILURE;\ +} while(0) + int main(int argc, char *argv[]) { - struct comm comm; - struct crystal cr; - struct array eList; - elm_data *data; + MPI_Init(&argc, &argv); + int rank,size; + MPI_Comm_rank(MPI_COMM_WORLD,&rank); + MPI_Comm_size(MPI_COMM_WORLD,&size); + + if(argc!=3){ + if(rank==0) + printf("Usage: %s <#nread> \n",argv[0]); + EXIT_ERROR(); + } - int ierr; - int e, n, nel, nv; - int options[3]; + int n_read = atoi(argv[1]); + char* mesh_name = argv[2]; - MPI_Init(&argc, &argv); - comm_init(&comm, MPI_COMM_WORLD); + char geom_name[BUFSIZ]; + strncpy(geom_name,mesh_name,BUFSIZ); + strncat(geom_name,".re2",BUFSIZ); + + char conn_name[BUFSIZ]; + strncpy(conn_name,mesh_name,BUFSIZ); + strncat(conn_name,".co2",BUFSIZ); int color = MPI_UNDEFINED; + if(rank \n"); - return EXIT_FAILURE; - } + long long *vl=NULL; + double *coord=NULL; + int nelt=0,ndim,nv; - int nRead = atoi(argv[1]); - char* conFile = argv[2]; + /* Read mesh data */ + if(color==1){ + struct comm comm; comm_init(&comm, comm_read); - if (comm.id < nRead) color = 1; - MPI_Comm comm_read; - MPI_Comm_split(comm.c, color, 0, &comm_read); + Mesh mesh; + read_geometry(&mesh,geom_name,&comm); + read_connectivity(mesh,conn_name,&comm); + get_vertex_ids(&vl,mesh); + get_vertex_coordinates(&coord,mesh); - ierr = 0; - struct con con = {}; - if (color != MPI_UNDEFINED) ierr = conRead(conFile, &con, comm_read); - if(ierr) goto quit; + ndim = get_mesh_dim(mesh); + nelt = get_mesh_nel(mesh); - comm_bcast(&comm, &con.nv, sizeof(int), 0); - nv = con.nv; - nel = con.nel; - int *part = (int*) malloc(nel * sizeof(int)); + mesh_free(mesh); + comm_free(&comm); + } + + MPI_Bcast(&ndim,1,MPI_INT,0,MPI_COMM_WORLD); + nv=(ndim==3)?8:4; + /* Partition the mesh */ + int options[3]; options[0] = 1; /* use custom options */ - options[1] = 3; /* debug level */ + options[1] = 2; /* debug level */ options[2] = 0; /* not used */ - ierr = parRSB_partMesh(part, con.vl, nel, nv, options, comm.c); - if(ierr) goto quit; + int *part=(int*)calloc(nelt,sizeof(int)); + int ierr=parRSB_partMesh(part,vl,coord,nelt,nv,options,MPI_COMM_WORLD); + if(ierr){ + if(vl) free(vl); + if(coord) free(coord); + if(part) free(part); + EXIT_ERROR(); + } - /* redistribute data */ - array_init(elm_data, &eList, nel), eList.n = nel; - for(data = eList.ptr, e = 0; e < nel; ++e) { - data[e].proc = part[e]; - data[e].id = con.el[e]; - for(n = 0; n < nv; ++n) { - data[e].vtx[n] = con.vl[e * nv + n]; + /* Redistribute data */ + struct array elements; + array_init(elm_data,&elements,nelt); + elm_data *data; + int e,n; + for(data=elements.ptr,e=0; e +#include + +int read_co2_mesh(Mesh *mesh_,char *fname,struct comm *c){ + comm_ext comm=c->c; + int size=c->np; + int rank=c->id; + + MPI_File file; + int err=MPI_File_open(comm,fname,MPI_MODE_RDONLY,MPI_INFO_NULL,&file); + if(err){ + if(rank==0) + printf("Error opening %s for reading.\n",fname); + return 1; + } + + char *buf=(char*)calloc(GC_CO2_HEADER_LEN+1,sizeof(char)); + MPI_Status st; + err=MPI_File_read_all(file,buf,GC_CO2_HEADER_LEN,MPI_BYTE,&st); + + int nelgt,nelgv,nVertex; + char version[6]; + sscanf(buf,"%5s%12d%12d%12d",version,&nelgt,&nelgv,&nVertex); + +#if defined(GENMAP_DEBUG) + printf("rank %d: %s %d %d %d\n",rank,version,nelgt,nelgv,nVertex); +#endif + + //TODO: Assert version + int nDim=(nVertex==8)?3:2; + int nelt=nelgt/size; + int nrem=nelgt-nelt*size; + nelt+=(rank>(size-1-nrem)? 1: 0); + + float byte_test; + MPI_File_read_all(file,&byte_test,4,MPI_BYTE,&st); + if(fabs(byte_test-6.543210)>1e-7){ + if(rank==0) + printf("ERROR byte_test failed! %f\n",byte_test); + return 1; + } + + // Initialize the mesh structure + mesh_init(mesh_,nelt,nDim); Mesh mesh=*mesh_; + mesh->nelgt=nelgt; + mesh->nelgv=nelgv; + mesh->nelt=nelt; + mesh->nDim=nDim; + + slong out[2][1],bfr[2][1],in[1]; + in[0]=nelt; + comm_scan(out,c,gs_long,gs_add,&in,1,bfr); + slong start=out[0][0]; + + int read_size=nelt*(nVertex+1)*sizeof(int); + int header_size=GC_CO2_HEADER_LEN+sizeof(float); + if(rank==0) + read_size+=header_size; + + buf=(char*)realloc(buf,read_size*sizeof(char)); + err=MPI_File_read_ordered(file,buf,read_size,MPI_BYTE,&st); + err=MPI_File_close(&file); + + char *buf0=buf; + if(rank==0) + buf0+=header_size; + + Element ptr=mesh->elements.ptr; + int i,j,tmp1,tmp2; + for(i=0;ielements.n=nelt*nVertex; + + free(buf); + + return 0; +} + +int read_connectivity(Mesh mesh,char *fname,struct comm *c){ + comm_ext comm=c->c; + int rank=c->id; + int size=c->np; + + MPI_File file; + int err=MPI_File_open(comm,fname,MPI_MODE_RDONLY,MPI_INFO_NULL,&file); + if(err!=0){ + if(rank==0) + printf("Error opening %s for reading.\n",fname); + return 1; + } + + char *buf=(char*)calloc(GC_CO2_HEADER_LEN+1,sizeof(char)); + MPI_Status st; + err=MPI_File_read_all(file,buf,GC_CO2_HEADER_LEN,MPI_BYTE,&st); + + int nelgt,nelgv,nVertex; + char version[6]; + sscanf(buf,"%5s%12d%12d%12d",version,&nelgt,&nelgv,&nVertex); + + //TODO: Assert version + int nDim=(nVertex==8)?3:2; + int nelt=nelgt/size; + int nrem=nelgt-nelt*size; + nelt+=(rank>(size-1-nrem)? 1: 0); + + float byte_test; + MPI_File_read_all(file,&byte_test,4,MPI_BYTE,&st); + if(fabs(byte_test-6.543210)>1e-7){ + if(rank==0) + printf("ERROR byte_test failed! %f\n",byte_test); + return 1; + } + + // Check for the same .re2 mesh + assert(mesh->nDim==nDim); + assert(mesh->nelgt==nelgt); + assert(mesh->nelgv==nelgv); + assert(mesh->nelt==nelt); + assert(mesh->elements.n==nelt*nVertex); + +#if defined(GENMAP_DEBUG) + printf("co2 %d: %s %d %d %d %d\n",rank,version,nelgt,nelgv,nelt,nDim); +#endif + + slong out[2][1],bfr[2][1],in[1]; + in[0]=nelt; + comm_scan(out,c,gs_long,gs_add,&in,1,bfr); + slong start=out[0][0]; + + int read_size=nelt*(nVertex+1)*sizeof(int); + int header_size=GC_CO2_HEADER_LEN+sizeof(float); + if(rank==0) + read_size+=header_size; + + buf=(char*)realloc(buf,read_size*sizeof(char)); + err=MPI_File_read_ordered(file,buf,read_size,MPI_BYTE,&st); + err=MPI_File_close(&file); + + char *buf0=buf; + if(rank==0) + buf0+=header_size; + + Point ptr=mesh->elements.ptr; + int i,j,tmp1,tmp2; + for(i=0;iid; + uint size=c->np; + MPI_Comm comm=c->c; + + int nVertex=mesh->nVertex; + int nDim=mesh->nDim; + sint nelt=mesh->nelt; + slong nelgt=mesh->nelgt; + slong nelgv=mesh->nelgv; + + int errs=0; + + MPI_File file; + int err=MPI_File_open(comm,fileName,\ + MPI_MODE_CREATE|MPI_MODE_WRONLY,MPI_INFO_NULL,&file); + if(err){ + if(rank==0) + fprintf(stderr,"%s:%d Error opening file: %s for writing.\n", + __FILE__,__LINE__,fileName); + errs++; + MPI_Abort(comm,911); + } + + slong out[2][1],bfr[2][1],in=nelt; + comm_scan(out,c,gs_long,gs_add,&in,1,bfr); + slong start=out[0][0]; + + int writeSize=nelt*(nVertex+1)*sizeof(int); + int header_size=GC_CO2_HEADER_LEN+sizeof(float); + if(rank==0) writeSize+=header_size; + + char *buf=(char*)calloc(writeSize,sizeof(char)),*buf0=buf; + MPI_Status st; + if(rank==0){ + sprintf(buf0,"%5s%12d%12d%12d",version,(int)nelgt,\ + (int)nelgv,nVertex); +#if defined(GENMAP_DEBUG) + printf("%5s%12d%12d%12d\n",version,(int)nelgt,(int)nelgv,nVertex); +#endif + memset(buf0+strlen(buf0),' ',GC_CO2_HEADER_LEN-strlen(buf0)); + buf0[GC_CO2_HEADER_LEN]='\0'; + buf0+=GC_CO2_HEADER_LEN; + memcpy(buf0,&test,sizeof(float)),buf0+=sizeof(float); + } + + Point ptr=mesh->elements.ptr; + int i,k,temp; + for(i=0;ielementId+1; + WRITE_INT(buf0,temp); buf0+=sizeof(int); +#if defined(GENMAP_DEBUG) + printf("%d ",temp); +#endif + for(k=0;kglobalId+1; + WRITE_INT(buf0,temp); buf0+=sizeof(int); +#if defined(GENMAP_DEBUG) + printf("%d ",temp); +#endif + ptr++; + } +#if defined(GENMAP_DEBUG) + printf("\n"); +#endif + } + + err=MPI_File_write_ordered(file,buf,writeSize,MPI_BYTE,&st); + if(err) errs++; + + err=MPI_File_close(&file); + if(err) errs++; + + MPI_Barrier(comm); + free(buf); + + return errs; +} diff --git a/src/gencon/face-check.c b/src/gencon/face-check.c new file mode 100644 index 00000000..4e0ba840 --- /dev/null +++ b/src/gencon/face-check.c @@ -0,0 +1,245 @@ +#include + +#include +#include + +typedef struct{ + ulong *elements; + uint *offsets; + ulong *globalIds; + uint size; +} VToEMap; + +typedef struct{ + ulong elementId; +} ElementID; + +typedef struct{ + ulong procId; +} ProcID; + +VToEMap *getVToEMap(Mesh m,struct comm *c) +{ + sint nelt=m->nelt; + sint nv =m->nVertex; + + slong out[2][1],buf[2][1],in=nelt; + comm_scan(out,c,gs_long,gs_add,&in,1,buf); + ulong elemId=out[0][0]; + ulong sequenceId=elemId*nv; + + size_t size=nelt*nv; + struct array vertices; array_init(vertex,&vertices,size); + + // create (globalId, elementId) pairs and send them to + // globalId % np + Point ptr=m->elements.ptr; + sint i,j; + for(i=0; inp + }; + array_cat(vertex,&vertices,&t,1); + sequenceId++; + } + elemId++; + } + + buffer bfr; buffer_init(&bfr,1024); + sarray_sort_2(vertex,vertices.ptr,vertices.n,vertexId,1, + elementId,1,&bfr); + + struct array vtcsCmpct; array_init(vertex,&vtcsCmpct,10); + vertex* vPtr=vertices.ptr; + + if(vertices.n>0){ + vertex prev=vPtr[0]; + array_cat(vertex,&vtcsCmpct,&prev,1); + + for(i=1; ielements=calloc(a.n,sizeof(ulong)); + + uint nGIds=1,prev=0; + vertex *aPtr=a.ptr; + for(i=1; isize=nGIds; + map->globalIds=calloc(nGIds,sizeof(ulong)); + map->offsets=calloc(nGIds+1,sizeof(ulong)); + + map->elements[0] =aPtr[0].elementId; + map->globalIds[0]=aPtr[0].vertexId; + map->offsets[0] =0; + + prev=0; uint nOffsets=0; + for(i=1; iglobalIds[nOffsets]=aPtr[i].vertexId; + map->offsets[nOffsets]=prev=i; + } + map->elements[i]=aPtr[i].elementId; + } + map->offsets[++nOffsets]=a.n; + assert(nOffsets==nGIds); + + array_free(&a); + + return map; +} + +/* key must be present in globalIds */ +int getPosition(VToEMap *map,ulong key){ + ulong *globalIds=map->globalIds; + + int begin=0; + int end=map->size; + int mid; + while(begin<=end){ + mid=(begin+end)/2; + + if(key==globalIds[mid]) + return mid; + else if(keyglobalIds); + free(map->offsets); + free(map->elements); + free(map); +} + +int faceCheck(Mesh mesh,struct comm *c) +{ + VToEMap *map=getVToEMap(mesh,c); + + sint nelt=mesh->nelt; + sint ndim=mesh->nDim; + + int faces[GC_MAX_FACES][GC_MAX_FACE_VERTICES]; + if(ndim==3) + memcpy(faces,faces3D,GC_MAX_FACES*GC_MAX_FACE_VERTICES*sizeof(int)); + else + memcpy(faces,faces2D,GC_MAX_FACES*GC_MAX_FACE_VERTICES*sizeof(int)); + + Point ptr=mesh->elements.ptr; + int nf =(ndim==3)?6:4; + int nfv=(ndim==3)?4:2; + int nv =(ndim==3)?8:4; + + struct array shared; array_init(ElementID,&shared,200); + buffer bfr; buffer_init(&bfr,1024); + + int i,j,k,l; + for(i=0; isize && "Return index out of bounds"); + + ElementID elemId; + for(l=map->offsets[indx]; loffsets[indx+1]; l++){ + elemId.elementId=map->elements[l]; + array_cat(ElementID,&shared,&elemId,1); + } + } + + sarray_sort(ElementID,shared.ptr,shared.n,elementId,1,&bfr); + + ulong prev=0; int ncount=1; + ElementID *ptr=shared.ptr; + for(l=1; l(b) ? (a) : (b)) +#define READ_T(coords,buf,T,nVertex) do{\ + memcpy((coords),buf,sizeof(T)*nVertex);\ +} while(0) + +#define WRITE_INT(dest,val) do{\ + memcpy(dest,&(val),sizeof(int));\ +} while(0) + +// TODO: Use rsb_element here struct Point_private{ GenmapScalar dx; GenmapScalar x[GC_MAX_DIM]; @@ -22,6 +31,9 @@ struct Point_private{ slong globalId; }; +extern int faces3D[GC_MAX_FACES][GC_MAX_FACE_VERTICES]; +extern int faces2D[GC_MAX_FACES][GC_MAX_FACE_VERTICES]; + struct Face_private{ struct Point_private vertex[GC_MAX_FACE_VERTICES]; }; diff --git a/src/gencon/gencon.h b/src/gencon/gencon.h index c10de2cd..a4080af5 100644 --- a/src/gencon/gencon.h +++ b/src/gencon/gencon.h @@ -32,18 +32,23 @@ extern int PRE_TO_SYM_VERTEX[GC_MAX_VERTICES]; extern int PRE_TO_SYM_FACE[GC_MAX_FACES]; /* Mesh */ -int MeshInit(Mesh *m_,int nel,int nDim); +int mesh_init(Mesh *m,int nel,int nDim); Element MeshGetElements(Mesh m); -int MeshFree(Mesh m); +void get_vertex_ids(long long **ids,Mesh m); +void get_vertex_coordinates(double **coords,Mesh m); +int get_mesh_dim(Mesh m); +int get_mesh_nel(Mesh m); +int mesh_free(Mesh m); /* Read Nek5000 mesh files */ -int read_re2_mesh(Mesh *mesh,char *fname,struct comm *c); -int read_co2_file(Mesh mesh,char *fname,struct comm *c); -int write_co2_file(Mesh mesh,char *fname,struct comm *c); +int read_geometry(Mesh *mesh,char *fname,struct comm *c); +int read_connectivity(Mesh mesh,char *fname,struct comm *c); +int write_connectivity(Mesh mesh,char *fname,struct comm *c); int read_co2_mesh(Mesh *mesh,char *fname,struct comm *c); int findMinNeighborDistance(Mesh mesh); -int findSegments(Mesh mesh,struct comm *c,GenmapScalar tol); +int findSegments(Mesh mesh,struct comm *c,GenmapScalar tol,int verbose); +int faceCheck(Mesh mesh,struct comm *c); int setGlobalID(Mesh mesh,struct comm *c); int sendBack(Mesh mesh,struct comm *c); int matchPeriodicFaces(Mesh mesh,struct comm *c); diff --git a/src/gencon/geometry_io.c b/src/gencon/geometry_io.c new file mode 100644 index 00000000..a066e295 --- /dev/null +++ b/src/gencon/geometry_io.c @@ -0,0 +1,248 @@ +#include +#include + +#include +#include + +int PRE_TO_SYM_VERTEX[GC_MAX_VERTICES]={0,1,3,2,4,5,7,6}; +int PRE_TO_SYM_FACE[GC_MAX_FACES]={2,1,3,0,4,5}; + +int transferBoundaryFaces(Mesh mesh,struct comm *c){ + uint size=c->np; + + struct array *boundary=&mesh->boundary; + BoundaryFace ptr=boundary->ptr; + int nFaces=boundary->n; + + slong nelgt=mesh->nelgt; + sint nelt=nelgt/size; + sint nrem=nelgt-nelt*size; + slong N=(size-nrem)*nelt; + + sint i; slong eid; + for(i=0;iid; + int size=c->np; + MPI_Comm comm=c->c; + + char *buf=(char *)calloc(GC_RE2_HEADER_LEN+1,sizeof(char)); + MPI_Status st; + int err=MPI_File_read_all(file,buf,GC_RE2_HEADER_LEN,MPI_BYTE,&st); + + int nelgt,nelgv,nDim; + char version[6]; + sscanf(buf,"%5s %d %d %d",version,&nelgt,&nDim,&nelgv); + + //TODO: Assert version + int nVertex=(nDim==2)?4:8; + int nelt=nelgt/size; + int nrem=nelgt-nelt*size; + nelt+=(rank>(size-1-nrem)? 1: 0); + + float byte_test; + MPI_File_read_all(file,&byte_test,4,MPI_BYTE,&st); + if(fabs(byte_test-6.543210)>1e-7){ + if(rank==0) + printf("ERROR byte_test failed! %f\n",byte_test); + return 1; + } + + mesh_init(mesh_,nelt,nDim); + Mesh mesh=*mesh_; + mesh->nelgt=nelgt; + mesh->nelgv=nelgv; + mesh->nelt=nelt; + mesh->nDim=nDim; + +#if defined(GENMAP_DEBUG) + printf("re2 %d: %s %d %d %d %d\n",rank,version,nelgt,nelgv,nelt,nDim); +#endif + + free(buf); + + return 0; +} + +int readRe2Coordinates(Mesh mesh,MPI_File file,struct comm *c){ + uint rank=c->id; + uint size=c->np; + MPI_Comm comm=c->c; + + int nelt=mesh->nelt; + int nelgt=mesh->nelgt; + int nDim=mesh->nDim; + int nVertex=(nDim==2)?4:8; + + slong out[2][1],bfr[2][1]; + slong in=nelt; + comm_scan(out,c,gs_long,gs_add,&in,1,bfr); + slong start=out[0][0]; + + int elemDataSize=nVertex*nDim*sizeof(double)+sizeof(double); + int header_size=GC_RE2_HEADER_LEN+sizeof(float); + + /* calculate read size for element data on each MPI rank */ + int read_size=nelt*elemDataSize; + if(rank==0) read_size+=header_size; + + char *buf=(char*)calloc(read_size,sizeof(char)); + char *buf0=buf; + MPI_Status st; + int err=MPI_File_read_ordered(file,buf,read_size,MPI_BYTE,&st); + if(err) + return 1; + + if(rank==0) + buf0+=header_size; + + /* initialize array */ + uint nUnits=nelt*nVertex; + array_init(struct Point_private,&mesh->elements,nUnits); + Point ptr=mesh->elements.ptr; + + /* read elements for each rank */ + double x[GC_MAX_VERTICES],y[GC_MAX_VERTICES],z[GC_MAX_VERTICES]; + int i,j,k; + for(i=0;ix[0]=x[j],ptr->x[1]=y[j]; + if(nDim==3) + ptr->x[2]=z[j]; + ptr->elementId =start+i; + ptr->sequenceId=nVertex*(start+i)+k; + ptr->origin =rank; + ptr++; + } + } + mesh->elements.n=nUnits; + +#if defined(GENMAP_DEBUG) + printf("io: rank=%d npts=%u\n",rank,nUnits); +#endif + + free(buf); + return 0; +} + +int readRe2Boundaries(Mesh mesh,MPI_File file,struct comm *c){ + uint rank=c->id; + uint size=c->np; + MPI_Comm comm=c->c; + + int nelt=mesh->nelt; + int nelgt=mesh->nelgt; + int nDim=mesh->nDim; + int nVertex=mesh->nVertex; + + int elemDataSize=nVertex*nDim*sizeof(double)+sizeof(double); + int header_size=GC_RE2_HEADER_LEN+sizeof(float); + + MPI_Status st; + char bufL[8]; + + /* calculate offset for the curve side data */ + MPI_Offset curveOffset=header_size+nelgt*elemDataSize; + if(rank==0) + MPI_File_read_at(file,curveOffset,bufL,sizeof(long),\ + MPI_BYTE,&st); + MPI_Bcast(bufL,sizeof(long),MPI_BYTE,0,comm); + + double ncurvesD; READ_T(&ncurvesD,bufL,long,1); + long ncurves=ncurvesD; + + /* calculate offset for boundary conditions data */ + MPI_Offset boundaryOffset=curveOffset+sizeof(long)+\ + sizeof(long)*8*ncurves; + if(rank==0) + MPI_File_read_at(file,boundaryOffset,bufL,sizeof(long),\ + MPI_BYTE,&st); + MPI_Bcast(bufL,sizeof(long),MPI_BYTE,0,comm); + + double nbcsD; READ_T(&nbcsD,bufL,long,1); long nbcs=nbcsD; + + int nbcsLocal=nbcs/size,nrem=nbcs-nbcsLocal*size; + nbcsLocal+=(rank>=(size-nrem) ? 1 : 0); + + slong out[2][1],bfr[2][1],in=nbcsLocal; + comm_scan(out,c,gs_long,gs_add,&in,1,bfr); + slong start=out[0][0]; + + int offset=boundaryOffset+sizeof(long)+start*8*sizeof(long); + int read_size=nbcsLocal*sizeof(long)*8; + char *buf=calloc(read_size,sizeof(char)),*buf0=buf; + MPI_File_read_at_all(file,offset,buf,read_size,MPI_BYTE,&st); + + double tmp[5]; + char cbc[4]; + struct Boundary_private boundary; + sint i; + for(i=0;iboundary,&boundary,1); + } + } + free(buf); +} + +int read_geometry(Mesh *mesh_,char *fname,struct comm *c){ + int nelt,nDim,nVertex; + int errs=0; + + uint rank=c->id; + uint size=c->np; + MPI_Comm comm=c->c; + + MPI_File file; + int err=MPI_File_open(comm,fname,MPI_MODE_RDONLY,MPI_INFO_NULL,&file); + if(err!=0){ + if(rank==0) + printf("%s:%d Error opening file: %s\n",__FILE__,__LINE__,fname); + return 1; + } + + readRe2Header(mesh_,file,c); + Mesh mesh=*mesh_; + readRe2Coordinates(mesh,file,c); + readRe2Boundaries(mesh,file,c); + transferBoundaryFaces(mesh,c); + + err=MPI_File_close(&file); + if(err) errs++; + + MPI_Barrier(comm); + + return errs; +} diff --git a/src/gencon/io.c b/src/gencon/io.c deleted file mode 100644 index 52d510f3..00000000 --- a/src/gencon/io.c +++ /dev/null @@ -1,509 +0,0 @@ -#include -#include - -#include -#include - -#define readT(coords,buf,T,nVertex) do{\ - memcpy((coords),buf,sizeof(T)*nVertex);\ -} while(0) - -#define writeInt(dest,val) do{\ - memcpy(dest,&(val),sizeof(int));\ -} while(0) - -int PRE_TO_SYM_VERTEX[GC_MAX_VERTICES]={0,1,3,2,4,5,7,6}; -int PRE_TO_SYM_FACE[GC_MAX_FACES]={2,1,3,0,4,5}; - -int transferBoundaryFaces(Mesh mesh,struct comm *c){ - uint size=c->np; - - struct array *boundary=&mesh->boundary; - BoundaryFace ptr=boundary->ptr; - int nFaces=boundary->n; - - slong nelgt=mesh->nelgt; - sint nelt=nelgt/size; - sint nrem=nelgt-nelt*size; - slong N=(size-nrem)*nelt; - - sint i; slong eid; - for(i=0;iid; - uint size=c->np; - MPI_Comm comm=c->c; - - char *buf=(char *)calloc(GC_RE2_HEADER_LEN+1,sizeof(char)); - if(rank==0){ - err=MPI_File_read(file,buf,GC_RE2_HEADER_LEN,MPI_BYTE,&st); - if(err) return 1; - } - MPI_Bcast(buf,GC_RE2_HEADER_LEN,MPI_BYTE,0,comm); - if(rank==0){ -#if defined(GENMAP_DEBUG) - printf("header: %s\n",buf); -#endif - } - sscanf(buf,"%5s %d %d %d",version,&nelgt,&nDim,&nelgv); - - int nVertex=(nDim==2)?4:8; - int nelt=nelgt/size,nrem=nelgt-nelt*size; - nelt+=(rank>=(size-nrem) ? 1: 0); - - MeshInit(mesh_,nelt,nDim); Mesh mesh=*mesh_; - mesh->nelgt=nelgt; - mesh->nelgv=nelgv; - - if(rank==0){ -#if defined(GENMAP_DEBUG) - printf("ndim,nvertex,nneighbors,nelgt,nelt:%d %d %d %lld %d\n", - mesh->nDim,mesh->nVertex,mesh->nNeighbors,mesh->nelgt,mesh->nelt); -#endif - } - - free(buf); - - return 0; -} - -int readRe2Coordinates(Mesh mesh,MPI_File file,struct comm *c){ - uint rank=c->id; - uint size=c->np; - MPI_Comm comm=c->c; - - int nelt=mesh->nelt; - int nelgt=mesh->nelgt; - int nDim=mesh->nDim; - int nVertex=mesh->nVertex; - - slong out[2][1],buff[2][1],in=nelt; - comm_scan(out,c,gs_long,gs_add,&in,1,buff); - slong start=out[0][0]; - - int elemDataSize=nVertex*nDim*sizeof(double)+sizeof(double); - int headerSize=GC_RE2_HEADER_LEN+sizeof(float); - - /* calculate read size for element data on each MPI rank */ - int readSize=nelt*elemDataSize; - if(rank==0) readSize+=headerSize; - - char *buf=(char*)calloc(readSize,sizeof(char)),*buf0=buf; - MPI_Status st; - int err=MPI_File_read_ordered(file,buf,readSize,MPI_BYTE,&st); - if(err) return 1; - - if(rank==0) buf0+=headerSize; - - /* initialize array */ - uint nUnits=nelt*nVertex; - array_init(struct Point_private,&mesh->elements,nUnits); - Point ptr=mesh->elements.ptr; - - /* read elements for each rank */ - double x[GC_MAX_VERTICES],y[GC_MAX_VERTICES],z[GC_MAX_VERTICES]; - int i,j,k; - for(i=0;ix[0]=x[j],ptr->x[1]=y[j]; - if(nDim==3) - ptr->x[2]=z[j]; - ptr->elementId =start+i; - ptr->sequenceId=nVertex*(start+i)+k; - ptr->origin =rank; - ptr++; - } - } - mesh->elements.n=nUnits; - -#if defined(GENMAP_DEBUG) - printf("io: rank=%d npts=%u\n",rank,nUnits); -#endif - - free(buf); - return 0; -} - -int readRe2Boundaries(Mesh mesh,MPI_File file,struct comm *c){ - uint rank=c->id; - uint size=c->np; - MPI_Comm comm=c->c; - - int nelt=mesh->nelt; - int nelgt=mesh->nelgt; - int nDim=mesh->nDim; - int nVertex=mesh->nVertex; - - int elemDataSize=nVertex*nDim*sizeof(double)+sizeof(double); - int headerSize=GC_RE2_HEADER_LEN+sizeof(float); - - MPI_Status st; - char bufL[8]; - - /* calculate offset for the curve side data */ - MPI_Offset curveOffset=headerSize+nelgt*elemDataSize; - if(rank==0) - MPI_File_read_at(file,curveOffset,bufL,sizeof(long),\ - MPI_BYTE,&st); - MPI_Bcast(bufL,sizeof(long),MPI_BYTE,0,comm); - - double ncurvesD; readT(&ncurvesD,bufL,long,1); - long ncurves=ncurvesD; - - /* calculate offset for boundary conditions data */ - MPI_Offset boundaryOffset=curveOffset+sizeof(long)+\ - sizeof(long)*8*ncurves; - if(rank==0) - MPI_File_read_at(file,boundaryOffset,bufL,sizeof(long),\ - MPI_BYTE,&st); - MPI_Bcast(bufL,sizeof(long),MPI_BYTE,0,comm); - - double nbcsD; readT(&nbcsD,bufL,long,1); long nbcs=nbcsD; - - int nbcsLocal=nbcs/size,nrem=nbcs-nbcsLocal*size; - nbcsLocal+=(rank>=(size-nrem) ? 1 : 0); - - slong out[2][1],buff[2][1],in=nbcsLocal; - comm_scan(out,c,gs_long,gs_add,&in,1,buff); - slong start=out[0][0]; - - int offset=boundaryOffset+sizeof(long)+start*8*sizeof(long); - int readSize=nbcsLocal*sizeof(long)*8; - char *buf=calloc(readSize,sizeof(char)),*buf0=buf; - MPI_File_read_at_all(file,offset,buf,readSize,MPI_BYTE,&st); - - double tmp[5]; - char cbc[4]; - struct Boundary_private boundary; - sint i; - for(i=0;iboundary,&boundary,1); - } - } - free(buf); -} - -int read_re2_mesh(Mesh *mesh_,char *fileName,struct comm *c){ - int nelt,nDim,nVertex; - int errs=0; - - uint rank=c->id; - uint size=c->np; - MPI_Comm comm=c->c; - - MPI_File file; - int err=MPI_File_open(comm,fileName,MPI_MODE_RDONLY,\ - MPI_INFO_NULL,&file); - if(err){ - if(rank==0) - fprintf(stderr,"%s:%d Error opening file: %s\n", - __FILE__,__LINE__,fileName); - errs++; - MPI_Abort(comm,911); - } - - readRe2Header(mesh_,file,c); - Mesh mesh=*mesh_; - readRe2Coordinates(mesh,file,c); - readRe2Boundaries(mesh,file,c); - transferBoundaryFaces(mesh,c); - - err=MPI_File_close(&file); - if(err) errs++; - - MPI_Barrier(comm); - - return errs; -} - -int read_co2_mesh(Mesh *mesh_,char *fname,struct comm *c){ - comm_ext comm=c->c; - int size=c->np; - int rank=c->id; - - MPI_File file; - int err=MPI_File_open(comm,fname,MPI_MODE_RDONLY,MPI_INFO_NULL,&file); - if(err){ - if(rank==0) - fprintf(stderr,"%s:%d Error opening file: %s for reading.\n", - __FILE__,__LINE__,fname); - MPI_Abort(comm,911); - } - - char *buf=(char*)calloc(GC_CO2_HEADER_LEN+1,sizeof(char)); - MPI_Status st; - if(rank==0){ - err=MPI_File_read(file,buf,GC_CO2_HEADER_LEN,MPI_BYTE,&st); - if(err) return 1; - } - - MPI_Bcast(buf,GC_CO2_HEADER_LEN,MPI_BYTE,0,comm); - - int nelgt,nelgv,nVertex; - char version[6]; - sscanf(buf,"%5s%12d%12d%12d",version,&nelgt,&nelgv,&nVertex); - -#if defined(GENMPA_DEBUG) - if(rank==0) - printf("%s %d %d %d\n",version,nelgt,nelgv,nVertex); -#endif - - //TODO: Assert version - int nelt=nelgt/size,nrem=nelgt-nelt*size; - nelt+=(rank>=(size-nrem) ? 1: 0); - - int nDim=(nVertex==8)?3:2; - - // Initialize the mesh structure - MeshInit(mesh_,nelt,nDim); Mesh mesh=*mesh_; - mesh->nelgt=nelgt; - mesh->nelgv=nelgv; - -#if defined(GENMPA_DEBUG) - if(rank==0) - printf("ndim/nvertex/nelgt/nelgv/nelt: %d/%d/%d/%d/%d\n", - nDim,nVertex,nelgt,nelgv,nelt); -#endif - - slong out[2][1],buff[2][1],in[1]; - in[0]=nelt; - comm_scan(out,c,gs_long,gs_add,&in,1,buff); - slong start=out[0][0]; - - int readSize=nelt*(nVertex+1)*sizeof(int); - int headerSize=GC_CO2_HEADER_LEN+sizeof(float); - if(rank==0) - readSize+=headerSize; - - buf=(char*)realloc(buf,readSize*sizeof(char)); - err=MPI_File_read_ordered(file,buf,readSize,MPI_BYTE,&st); - err=MPI_File_close(&file); - - char *buf0=buf; - if(rank==0) buf0+=headerSize; - - Element ptr=mesh->elements.ptr; - int i,j,tmp1,tmp2; - for(i=0;ielements.n=nelt*nVertex; - - free(buf); - - return 0; -} - -int read_co2_file(Mesh mesh,char *fname,struct comm *c){ - comm_ext comm=c->c; - int rank=c->id; - int size=c->np; - - MPI_File file; - int err=MPI_File_open(comm,fname,MPI_MODE_RDONLY,MPI_INFO_NULL,&file); - if(err){ - if(rank==0) - fprintf(stderr,"%s:%d Error opening file: %s for reading.\n", - __FILE__,__LINE__,fname); - MPI_Abort(comm,911); - } - - char *buf=(char*)calloc(GC_CO2_HEADER_LEN+1,sizeof(char)); - MPI_Status st; - - if(rank==0){ - err=MPI_File_read(file,buf,GC_CO2_HEADER_LEN,MPI_BYTE,&st); - if(err) return 1; - } - - MPI_Bcast(buf,GC_CO2_HEADER_LEN,MPI_BYTE,0,comm); - - int nelgt,nelgv,nVertex; - char version[6]; - sscanf(buf,"%5s%12d%12d%12d",version,&nelgt,&nelgv,&nVertex); - -#if defined(GENMPA_DEBUG) - if(rank==0) - printf("%s %d %d %d\n",version,nelgt,nelgv,nVertex); -#endif - - //TODO: Assert version - int nelt=nelgt/size,nrem=nelgt-nelt*size; - nelt+=(ranknDim==nDim); - assert(mesh->nVertex==nVertex); - assert(mesh->nNeighbors==nDim); - assert(mesh->nelgt==nelgt); - assert(mesh->nelgv==nelgv); - assert(mesh->nelt==nelt); - assert(mesh->elements.n==nelt*nVertex); - -#if defined(GENMPA_DEBUG) - if(rank==0) - printf("ndim/nvertex/nelgt/nelgv/nelt: %d/%d/%d/%d/%d\n", - nDim,nVertex,nelgt,nelgv,nelt); -#endif - - slong out[2][1],buff[2][1],in[1]; - in[0]=nelt; - comm_scan(out,c,gs_long,gs_add,&in,1,buff); - slong start=out[0][0]; - - int readSize=nelt*(nVertex+1)*sizeof(int); - int headerSize=GC_CO2_HEADER_LEN+sizeof(float); - if(rank==0) - readSize+=headerSize; - - buf=(char*)realloc(buf,readSize*sizeof(char)); - err=MPI_File_read_ordered(file,buf,readSize,MPI_BYTE,&st); - err=MPI_File_close(&file); - - char *buf0=buf; - if(rank==0) buf0+=headerSize; - - Point ptr=mesh->elements.ptr; - int i,j,tmp1,tmp2; - for(i=0;iid; - uint size=c->np; - MPI_Comm comm=c->c; - - int nVertex=mesh->nVertex; - int nDim=mesh->nDim; - sint nelt=mesh->nelt; - slong nelgt=mesh->nelgt; - slong nelgv=mesh->nelgv; - - int errs=0; - - MPI_File file; - int err=MPI_File_open(comm,fileName,\ - MPI_MODE_CREATE|MPI_MODE_WRONLY,MPI_INFO_NULL,&file); - if(err){ - if(rank==0) - fprintf(stderr,"%s:%d Error opening file: %s for writing.\n", - __FILE__,__LINE__,fileName); - errs++; - MPI_Abort(comm,911); - } - - slong out[2][1],buff[2][1],in=nelt; - comm_scan(out,c,gs_long,gs_add,&in,1,buff); - slong start=out[0][0]; - - int writeSize=nelt*(nVertex+1)*sizeof(int); - int headerSize=GC_CO2_HEADER_LEN+sizeof(float); - if(rank==0) writeSize+=headerSize; - - char *buf=(char*)calloc(writeSize,sizeof(char)),*buf0=buf; - MPI_Status st; - if(rank==0){ - sprintf(buf0,"%5s%12d%12d%12d",version,(int)nelgt,\ - (int)nelgv,nVertex); -#if defined(GENMAP_DEBUG) - printf("%5s%12d%12d%12d\n",version,(int)nelgt,(int)nelgv,nVertex); -#endif - memset(buf0+strlen(buf0),' ',GC_CO2_HEADER_LEN-strlen(buf0)); - buf0[GC_CO2_HEADER_LEN]='\0'; - buf0+=GC_CO2_HEADER_LEN; - memcpy(buf0,&test,sizeof(float)),buf0+=sizeof(float); - } - - Point ptr=mesh->elements.ptr; - int i,k,temp; - for(i=0;ielementId+1; - writeInt(buf0,temp); buf0+=sizeof(int); -#if defined(GENMAP_DEBUG) - printf("%d ",temp); -#endif - for(k=0;kglobalId+1; - writeInt(buf0,temp); buf0+=sizeof(int); -#if defined(GENMAP_DEBUG) - printf("%d ",temp); -#endif - ptr++; - } -#if defined(GENMAP_DEBUG) - printf("\n"); -#endif - } - - err=MPI_File_write_ordered(file,buf,writeSize,MPI_BYTE,&st); - if(err) errs++; - - err=MPI_File_close(&file); - if(err) errs++; - - MPI_Barrier(comm); - free(buf); - - return errs; -} - -#undef readT -#undef writeInt diff --git a/src/gencon/mesh.c b/src/gencon/mesh.c index 28e631ba..66e14600 100644 --- a/src/gencon/mesh.c +++ b/src/gencon/mesh.c @@ -1,7 +1,7 @@ #include #include -int MeshInit(Mesh *m_,int nel,int nDim){ +int mesh_init(Mesh *m_,int nel,int nDim){ GenmapMalloc(1,m_); Mesh m=*m_; @@ -20,7 +20,7 @@ Element MeshGetElements(Mesh m){ return (Element) m->elements.ptr; } -int MeshFree(Mesh m){ +int mesh_free(Mesh m){ array_free(&m->elements); array_free(&m->boundary); @@ -28,3 +28,47 @@ int MeshFree(Mesh m){ return 0; } + +void get_vertex_ids(long long **vertex_ids_,Mesh mesh){ + int nelt=mesh->nelt; + int nv=(mesh->nDim==3)?8:4; + + GenmapMalloc(nelt*nv,vertex_ids_); + long long *vertex_ids=*vertex_ids_; + + Point ptr=mesh->elements.ptr; + int e,v,count=0; + for(e=0; enelt; + int ndim=mesh->nDim; + int nv=(ndim==3)?8:4; + + GenmapMalloc(nelt*nv*ndim,coords_); + double *coords=*coords_; + + Point ptr=mesh->elements.ptr; + int e,v,d,count=0; + for(e=0; enDim; +} + +int get_mesh_nel(Mesh mesh){ + return mesh->nelt; +} diff --git a/src/gencon/segments.c b/src/gencon/segments.c index 6ab3e51c..733067f1 100644 --- a/src/gencon/segments.c +++ b/src/gencon/segments.c @@ -48,7 +48,7 @@ int mergeSegments(Mesh mesh,struct comm *c,int i,GenmapScalar tolSquared) return 0; } -int findSegments(Mesh mesh,struct comm *c,GenmapScalar tol){ +int findSegments(Mesh mesh,struct comm *c,GenmapScalar tol,int verbose){ int nDim=mesh->nDim,nVertex=mesh->nVertex; GenmapScalar tolSquared=tol*tol; @@ -89,9 +89,10 @@ int findSegments(Mesh mesh,struct comm *c,GenmapScalar tol){ comm_scan(out,&nonZeroRanks,gs_long,gs_add,in,1,buff); slong start=out[0][0]; -#if defined(GENMAP_DEBUG) - printf("segments: rank=%d npts=%u start=%lld\n",rank,nPoints,start); -#endif + if(verbose>1){ + printf("segments: rank=%d npts=%u start=%lld\n",rank,nPoints,start); + fflush(stdout); + } sint i; for(i=0;i0) - count++; - - in[0]=count; - comm_allreduce(&nonZeroRanks,gs_long,gs_add,in,1,buff); - if(rank==0) - printf("locglob: %d %lld\n",dim+1,in[0]+1); -#endif + if(verbose>0){ + sint count=0; + for(i=0;i0) + count++; + + in[0]=count; + comm_allreduce(&nonZeroRanks,gs_long,gs_add,in,1,buff); + if(rank==0) + printf("locglob: %d %lld\n",dim+1,in[0]+1); + } } } diff --git a/src/genmap-binsort.c b/src/genmap-binsort.c deleted file mode 100644 index 3a8f1540..00000000 --- a/src/genmap-binsort.c +++ /dev/null @@ -1,207 +0,0 @@ -#include "genmap-impl.h" - -#include -#include -#include -#include - -void GenmapFiedlerMinMax(GenmapHandle h, GenmapScalar *min, GenmapScalar *max) { - *min = 1; *max = -1; - - GenmapElements e = GenmapGetElements(h); - GenmapInt i; - for(i = 0; i < GenmapGetNLocalElements(h); i++) { - if(e[i].fiedler < *min) { - *min = e[i].fiedler; - } - if(e[i].fiedler > *max) { - *max = e[i].fiedler; - } - } - - GenmapGop(GenmapGetLocalComm(h), min, 1, GENMAP_SCALAR, GENMAP_MIN); - GenmapGop(GenmapGetLocalComm(h), max, 1, GENMAP_SCALAR, GENMAP_MAX); -} - -void GenmapGlobalIdMinMax(GenmapHandle h, GenmapLong *min, GenmapLong *max) { - *min = LONG_MAX; *max = LONG_MIN; - - GenmapElements e = GenmapGetElements(h); - GenmapInt i; - for(i = 0; i < GenmapGetNLocalElements(h); i++) { - if(e[i].globalId0 < *min) { - *min = e[i].globalId0; - } - if(e[i].globalId0 > *max) { - *max = e[i].globalId0; - } - } - - GenmapGop(GenmapGetLocalComm(h), min, 1, GENMAP_SCALAR, GENMAP_MIN); - GenmapGop(GenmapGetLocalComm(h), max, 1, GENMAP_SCALAR, GENMAP_MAX); -} - -GenmapInt GenmapSetFiedlerBin(GenmapHandle h) { - GenmapScalar min, max; - GenmapFiedlerMinMax(h, &min, &max); - GenmapScalar range = max - min; - - int np = GenmapCommSize(GenmapGetLocalComm(h)); - GenmapInt nbins = np; - GenmapInt lelt = GenmapGetNLocalElements(h); - GenmapElements elements = GenmapGetElements(h); - - GenmapElements p, e; - for(p = elements, e = p + lelt; p != e; p++) { - int id; - for(id = 0; id < np; id++) { - GenmapScalar start = min + (range * id) / nbins; - GenmapScalar end = min + (range * (id + 1)) / nbins; - if(start <= p->fiedler && p->fiedler < end) { - p->proc = id; - break; - } - } - if(id == np) p->proc = np - 1; - } - - return 0; -} - -GenmapInt GenmapSetGlobalIdBin(GenmapHandle h) { - GenmapLong min, max; - GenmapGlobalIdMinMax(h, &min, &max); - GenmapLong range = max - min; - - GenmapInt np = GenmapCommSize(GenmapGetLocalComm(h)); - GenmapInt nbins = np; - GenmapInt lelt = GenmapGetNLocalElements(h); - GenmapElements elements = GenmapGetElements(h); - - GenmapElements p, e; - for(p = elements, e = p + lelt; p != e; p++) { - GenmapInt id; - for(id = 0; id < np; id++) { - GenmapLong start = min + (range * id) / nbins; - GenmapLong end = min + (range * (id + 1)) / nbins; - if(start <= p->globalId0 && p->globalId0 < end) { - p->proc = id; - break; - } - } - if(id == np) p->proc = np - 1; - } - - return 0; -} - -void GenmapAssignBins(GenmapHandle h, int field, buffer *buf0) { - GenmapElements elements = GenmapGetElements(h); - GenmapInt lelt = GenmapGetNLocalElements(h); - - if(field == GENMAP_FIEDLER) { - sarray_sort(struct GenmapElement_private, elements, (GenmapUInt)lelt, fiedler, - TYPE_DOUBLE, buf0); - GenmapSetFiedlerBin(h); - } else if(GENMAP_GLOBALID) { - sarray_sort(struct GenmapElement_private, elements, (GenmapUInt)lelt, - globalId0, TYPE_LONG, buf0); - GenmapSetGlobalIdBin(h); - } -} - -void GenmapTransferToBins(GenmapHandle h, int field, buffer *buf0) { - GenmapElements elements = GenmapGetElements(h); - GenmapInt lelt = GenmapGetNLocalElements(h); - - if(field == GENMAP_FIEDLER) { - sarray_transfer(struct GenmapElement_private, &(h->elementArray), proc, 0, - &(h->cr)); - GenmapScan(h, GenmapGetLocalComm(h)); - elements = GenmapGetElements(h); - lelt = GenmapGetNLocalElements(h); - sarray_sort(struct GenmapElement_private, elements, (GenmapUInt)lelt, fiedler, - TYPE_DOUBLE, buf0); -/* - sarray_sort_2(struct GenmapElement_private, elements, (GenmapUInt)lelt, fiedler, - TYPE_DOUBLE, globalId0, TYPE_LONG, buf0); -*/ - } else if(field == GENMAP_GLOBALID) { - sarray_transfer(struct GenmapElement_private, &(h->elementArray), proc, 0, - &(h->cr)); - GenmapScan(h, GenmapGetLocalComm(h)); - elements = GenmapGetElements(h); - lelt = GenmapGetNLocalElements(h); - sarray_sort(struct GenmapElement_private, elements, (GenmapUInt)lelt, - globalId0, TYPE_LONG, buf0); - } -} - -void GenmapSplitByGlobalId(GenmapHandle h) { - GenmapLong start = GenmapGetLocalStartIndex(h); - GenmapLong nel = GenmapGetNGlobalElements(h); - GenmapInt id = GenmapCommRank(GenmapGetLocalComm(h)); - GenmapInt np = GenmapCommSize(GenmapGetLocalComm(h)); - GenmapInt lelt = GenmapGetNLocalElements(h); - GenmapElements elements = GenmapGetElements(h); - - GenmapInt pNel = (GenmapInt)(nel / np); - GenmapInt nrem = (GenmapInt)(nel - pNel * np); - GenmapInt idCount = 0; - while(idCount * pNel + ((idCount < nrem) ? idCount : nrem) < start) - idCount++; - - GenmapLong upLimit = idCount * pNel + ((idCount < nrem) ? idCount : nrem); - GenmapLong downLimit = start; - do { - GenmapInt end = upLimit - start < lelt ? (GenmapInt)(upLimit - start) : lelt; - GenmapInt i; - for(i = (GenmapInt)(downLimit - start); i < end; i++) - elements[i].proc = idCount - 1; - downLimit = upLimit; - idCount++; - upLimit = idCount * pNel + ((idCount < nrem) ? idCount : nrem); - } while(downLimit - start < lelt); -} - -void GenmapSplitByMedian(GenmapHandle h) { - GenmapLong start = GenmapGetLocalStartIndex(h); - GenmapLong nel = GenmapGetNGlobalElements(h); - GenmapInt id = GenmapCommRank(GenmapGetLocalComm(h)); - GenmapInt np = GenmapCommSize(GenmapGetLocalComm(h)); - GenmapInt lelt = GenmapGetNLocalElements(h); - GenmapElements elements = GenmapGetElements(h); - - GenmapInt pNel = (GenmapInt)(nel / np); - GenmapInt nrem = (GenmapInt)(nel - pNel * np); - GenmapInt idCount = 0; - while(idCount * pNel + ((idCount < nrem) ? idCount : nrem) < start) - idCount++; - - GenmapLong upLimit = idCount * pNel + ((idCount < nrem) ? idCount : nrem); - GenmapLong downLimit = start; - do { - GenmapInt end = upLimit - start < lelt ? (GenmapInt)(upLimit - start) : lelt; - GenmapInt i; - for(i = (GenmapInt)(downLimit - start); i < end; i++) - elements[i].proc = idCount - 1; - downLimit = upLimit; - idCount++; - upLimit = idCount * pNel + ((idCount < nrem) ? idCount : nrem); - } while(downLimit - start < lelt); -} - -void GenmapBinSort(GenmapHandle h, int field, buffer *buf0) { - GenmapElements elements = GenmapGetElements(h); - GenmapInt lelt = GenmapGetNLocalElements(h); - - GenmapAssignBins(h, field, buf0); - GenmapTransferToBins(h, field, buf0); - GenmapScan(h, GenmapGetLocalComm(h)); - if(field == GENMAP_FIEDLER) { - GenmapSplitByMedian(h); - } else if(field == GENMAP_GLOBALID) { - GenmapSplitByGlobalId(h); - } - GenmapTransferToBins(h, field, buf0); -} diff --git a/src/genmap-chelpers.c b/src/genmap-chelpers.c index c1d3bd6b..c5051962 100644 --- a/src/genmap-chelpers.c +++ b/src/genmap-chelpers.c @@ -8,6 +8,7 @@ #if defined __GLIBC__ #include + /* Obtain a backtrace and print it to stdout. */ void GenmapPrintStack(void) { void *bt[50]; @@ -21,7 +22,7 @@ void GenmapPrintStack(void) { } #else void GenmapPrintStack() {}; -#endif +#endif // defined __GLIBC__ double GenmapGetMaxRss() { struct rusage r_usage; @@ -33,28 +34,3 @@ double GenmapGetMaxRss() { return (double)(r_usage.ru_maxrss * 1024L); #endif } - -void set_stdout(char *f, int *sid, int flen) { - char *logfile = (char *) malloc((flen + 2 + 5 + 1) * sizeof(char)); - strncpy(logfile, f, flen); - int i; - for(i = flen - 1; i >= 0; i--) if(logfile[i] != ' ') break; - logfile[i + 1] = '\0'; - - int redirect = 0; - char *envvar; - - if(logfile[0] != '\0') { - redirect = 1; - } else if((envvar = getenv("NEK_LOGFILE")) != NULL) { - if(*sid >= 0) sprintf(logfile, "s%05d_", *sid); - strcat(logfile + strlen(logfile), envvar); - redirect = 1; - } - - if(redirect) { - printf("redirecting stdout to %s\n", logfile); - FILE *fp = freopen(logfile, "w+", stdout); - assert(fp != NULL); - } -} diff --git a/src/genmap-comm.c b/src/genmap-comm.c index fbef9663..fa1aa41e 100644 --- a/src/genmap-comm.c +++ b/src/genmap-comm.c @@ -1,9 +1,19 @@ #include "genmap-impl.h" -int GenmapCreateComm(GenmapComm *c_,GenmapCommExternal ce){ - GenmapMalloc(1,c_); GenmapComm c=*c_; - comm_init(&c->gsc, ce); c->gsh=NULL; c->M=NULL; c->b=NULL; +int GenmapCreateComm(GenmapComm *c_,comm_ext ce){ + GenmapMalloc(1,c_); + GenmapComm c=*c_; + comm_init(&c->gsc, ce); + + c->gsh=NULL; + c->M=NULL; + + c->gsw=NULL; + buffer_init(&c->buf,1024); + + c->b=NULL; + return 0; } @@ -14,6 +24,10 @@ int GenmapDestroyComm(GenmapComm c) { gs_free(c->gsh); if(c->M) csr_mat_free(c->M); + + if(c->gsw) + gs_free(c->gsw); + if(c->b) GenmapFree(c->b); @@ -31,19 +45,19 @@ int GenmapCommRank(GenmapComm c) { return (int) c->gsc.id; } -GenmapComm GenmapGetLocalComm(GenmapHandle h) { +GenmapComm GenmapGetLocalComm(genmap_handle h) { return h->local; } -void GenmapSetLocalComm(GenmapHandle h, GenmapComm c) { +void GenmapSetLocalComm(genmap_handle h, GenmapComm c) { h->local = c; } -GenmapComm GenmapGetGlobalComm(GenmapHandle h) { +GenmapComm GenmapGetGlobalComm(genmap_handle h) { return h->global; } -void GenmapSetGlobalComm(GenmapHandle h, GenmapComm c) { +void GenmapSetGlobalComm(genmap_handle h, GenmapComm c) { h->global = c; } @@ -75,8 +89,8 @@ int GenmapBcast(GenmapComm c, void *in, GenmapInt count, GenmapDataType type) { return MPI_Bcast(in, count, type, 0, c->gsc.c); } -void GenmapSplitComm(GenmapHandle h, GenmapComm *c, int bin) { - GenmapCommExternal local; +void GenmapSplitComm(genmap_handle h, GenmapComm *c, int bin) { + comm_ext local; int id = GenmapCommRank(*c); MPI_Comm_split((*c)->gsc.c, bin, id, &local); GenmapCrystalFinalize(h); @@ -86,22 +100,20 @@ void GenmapSplitComm(GenmapHandle h, GenmapComm *c, int bin) { GenmapCrystalInit(h, *c); } -int GenmapCrystalInit(GenmapHandle h, GenmapComm c) { +int GenmapCrystalInit(genmap_handle h, GenmapComm c) { crystal_init(&(h->cr), &(c->gsc)); return 0; } -int GenmapCrystalTransfer(GenmapHandle h, int field) { +int GenmapCrystalTransfer(genmap_handle h, int field) { if(field == GENMAP_ORIGIN) - sarray_transfer(struct GenmapElement_private, &(h->elementArray), origin, 0, - &(h->cr)); + sarray_transfer(struct rsb_element,h->elements,origin,0,&h->cr); else if(field == GENMAP_PROC) - sarray_transfer(struct GenmapElement_private, &(h->elementArray), proc, 0, - &(h->cr)); + sarray_transfer(struct rsb_element,h->elements,proc ,0,&h->cr); return 0; } -int GenmapCrystalFinalize(GenmapHandle h) { +int GenmapCrystalFinalize(genmap_handle h) { crystal_free(&(h->cr)); return 0; } diff --git a/src/genmap-components.c b/src/genmap-components.c new file mode 100644 index 00000000..f8ff4b90 --- /dev/null +++ b/src/genmap-components.c @@ -0,0 +1,52 @@ +#include +#include +#include +#include + +#include +#include + +/* Find the number of disconnected components */ +sint is_disconnected(struct comm *c,struct gs_data *gsh,buffer *buf, + uint nelt_,uint nv) +{ + slong nelt=nelt_; + + GenmapLong *p; + GenmapMalloc(nelt*nv,&p); + + slong out[2][1],buff[2][1]; + comm_scan(out,c,gs_long,gs_add,&nelt,1,buff); + slong nelg=out[1][0]; + + uint e; + for(e=0; eid==0){ + for(e=0; e0){ nnz1++; break; } + + if(dnnz0); + + GenmapFree(p); + + return (nnz1 +#include #include +#include -int GenmapSymTriDiagSolve(GenmapVector x, GenmapVector b, - GenmapVector alpha, - GenmapVector beta) { +int GenmapSymTriDiagSolve(GenmapVector x, GenmapVector b, GenmapVector alpha, GenmapVector beta) { assert((x->size == b->size) && (x->size == alpha->size)); assert(alpha->size == beta->size + 1); assert(b->size > 0); @@ -35,8 +35,7 @@ int GenmapSymTriDiagSolve(GenmapVector x, GenmapVector b, return 0; } -int GenmapInvPowerIter(GenmapVector eVector, GenmapVector alpha, - GenmapVector beta, GenmapVector init, GenmapInt iter) { +int GenmapInvPowerIter(GenmapVector eVector, GenmapVector alpha, GenmapVector beta, GenmapVector init, GenmapInt iter) { assert(alpha->size == beta->size + 1); assert(alpha->size == eVector->size); @@ -80,7 +79,7 @@ GenmapScalar GenmapSign(GenmapScalar a, GenmapScalar b) { return fabs(a) * m; } -int GenmapTQLI(GenmapHandle h, GenmapVector diagonal, GenmapVector upper, +int GenmapTQLI(genmap_handle h, GenmapVector diagonal, GenmapVector upper, GenmapVector **eVectors, GenmapVector *eValues) { assert(diagonal->size == upper->size + 1); @@ -202,3 +201,78 @@ int GenmapTQLI(GenmapHandle h, GenmapVector diagonal, GenmapVector upper, return 0; } + +int genmap_power(double *y, int N, double *A, int verbose){ + time_t t; + srand((unsigned)time(&t)); + + int i; + GenmapScalar norm=0.0; + for(i = 0; i < N; i++){ + y[i] = (rand()%50)/50.0; + norm += y[i]*y[i]; + } + + GenmapScalar normi = 1.0/sqrt(norm); + for(i=0; i0) + err = (sqrt(norm) - lambda)/lambda; + lambda = sqrt(norm); + + if(verbose){ + printf("\tInverse power: %02d %g\n", i, err); + } + + normi = 1.0/sqrt(norm); + for(j = 0; j < N; j++) + y[j] = Ay[j]*normi; + + if(fabs(err)<1.e-15) + break; + } + + GenmapFree(Ay); + + return i; +} + +int genmap_inverse_power(double *y, int N, double *A, int verbose){ + double *Ainv; + GenmapCalloc(N*N, &Ainv); + + int j, k; + for(j = 0; j < N; j++){ + for(k = 0; k < N; k++) + Ainv[j*N + k] = A[k*N + j]; + } + + matrix_inverse(N, Ainv); + + for(j = 0; j0){ #if defined(GENMAP_PAUL) @@ -31,17 +31,22 @@ int GenmapFiedlerRQI(GenmapHandle h,GenmapComm c,int maxIter,int global) } } - GenmapOrthogonalizebyOneVector(h,c,initVec,GenmapGetNGlobalElements(h)); - GenmapScalar rtr=GenmapDotVector(initVec,initVec); - GenmapGop(c,&rtr,1,GENMAP_SCALAR,GENMAP_SUM); - GenmapScalar rni=1.0/sqrt(rtr); - GenmapScaleVector(initVec,initVec,rni); - + int verbose=h->verbose_level; struct comm *gsc=&c->gsc; - metric_tic(gsc,LAPLACIANSETUP); + GenmapOrthogonalizebyOneVector(c,initVec,GenmapGetNGlobalElements(h)); + + GenmapScalar norm=GenmapDotVector(initVec,initVec); + GenmapGop(c,&norm,1,GENMAP_SCALAR,GENMAP_SUM); + if(verbose>0 && gsc->id==0) + printf("RQI, |init| = %g\n",sqrt(norm)); + + norm=1.0/sqrt(norm); + GenmapScaleVector(initVec,initVec,norm); + + metric_tic(gsc,LAPLACIANSETUP1); GenmapInitLaplacian(h,c); - metric_toc(gsc,LAPLACIANSETUP); + metric_toc(gsc,LAPLACIANSETUP1); metric_tic(gsc,PRECONSETUP); mgData d; mgSetup(c,c->M,&d); d->h=h; @@ -49,7 +54,8 @@ int GenmapFiedlerRQI(GenmapHandle h,GenmapComm c,int maxIter,int global) GenmapVector y; GenmapCreateZerosVector(&y,lelt); metric_tic(gsc,RQI); - int iter=rqi(h,c,d,initVec,maxIter,0,y); + int iter=rqi(h,c,d,initVec,max_iter,verbose,y); + metric_acc(NRQI,iter); metric_toc(gsc,RQI); mgFree(d); @@ -68,7 +74,7 @@ int GenmapFiedlerRQI(GenmapHandle h,GenmapComm c,int maxIter,int global) return iter; } -int GenmapFiedlerLanczos(GenmapHandle h,GenmapComm c,int maxIter, +int GenmapFiedlerLanczos(genmap_handle h,GenmapComm c,int max_iter, int global) { GenmapInt lelt = GenmapGetNLocalElements(h); @@ -100,21 +106,20 @@ int GenmapFiedlerLanczos(GenmapHandle h,GenmapComm c,int maxIter, } #endif - GenmapCreateVector(&alphaVec,maxIter); - GenmapCreateVector(&betaVec,maxIter-1); + GenmapCreateVector(&alphaVec,max_iter); + GenmapCreateVector(&betaVec,max_iter-1); GenmapVector *q = NULL; - GenmapOrthogonalizebyOneVector(h,c,initVec,GenmapGetNGlobalElements(h)); + GenmapOrthogonalizebyOneVector(c,initVec,GenmapGetNGlobalElements(h)); GenmapScalar rtr = GenmapDotVector(initVec, initVec); GenmapGop(c, &rtr, 1, GENMAP_SCALAR, GENMAP_SUM); GenmapScalar rni = 1.0 / sqrt(rtr); GenmapScaleVector(initVec, initVec, rni); #if defined(GENMAP_PAUL) - int iter=GenmapLanczosLegendary(h,c,initVec,maxIter,&q, - alphaVec,betaVec); + int iter=GenmapLanczosLegendary(h,c,initVec,max_iter,&q,alphaVec,betaVec); #else - int iter=GenmapLanczos(h,c,initVec,maxIter,&q,alphaVec,betaVec); + int iter=GenmapLanczos(h,c,initVec,max_iter,&q,alphaVec,betaVec); #endif GenmapVector evLanczos, evTriDiag; @@ -193,3 +198,112 @@ int GenmapFiedlerLanczos(GenmapHandle h,GenmapComm c,int maxIter, return iter; } + +#define write_T(dest,val,T,nunits) do{\ + memcpy(dest,&(val),sizeof(T)*nunits);\ + dest+=sizeof(T)*nunits;\ +} while(0) + +int GenmapFiedlerDump(const char *fname,genmap_handle h,GenmapComm comm) +{ + struct comm *c=&comm->gsc; + + MPI_File file; + int err=MPI_File_open(c->c,fname,MPI_MODE_CREATE|MPI_MODE_WRONLY, + MPI_INFO_NULL,&file); + uint rank=c->id; + if(err!=0 && rank==0){ + fprintf(stderr,"%s:%d Error opening file %s for writing.\n",__FILE__,__LINE__,fname); + return err; + } + + slong nelt=GenmapGetNLocalElements(h); + slong out[2][1],buf[2][1]; + comm_scan(out,c,gs_long,gs_add,&nelt,1,buf); + slong start=out[0][0]; + slong nelgt=out[1][0]; + + int ndim=(h->nv==8)?3:2; + uint write_size=(ndim+1)*nelt*sizeof(double); + if(rank==0) + write_size+=sizeof(long)+sizeof(int); // for nelgt and ndim + + char *pbuf,*pbuf0; + pbuf=pbuf0=(char*)calloc(write_size,sizeof(char)); + if(rank==0){ + write_T(pbuf0,nelgt,long,1); + write_T(pbuf0,ndim ,int ,1); + } + + GenmapElements elm=GenmapGetElements(h); + uint i; + for(i=0; ic); + + free(pbuf); + + return err; +} + +int GenmapVectorDump(const char *fname,GenmapScalar *y,uint size, + struct comm *c) +{ + MPI_File file; + int err=MPI_File_open(c->c,fname,MPI_MODE_CREATE|MPI_MODE_WRONLY, + MPI_INFO_NULL,&file); + uint rank=c->id; + if(err!=0 && rank==0){ + fprintf(stderr,"%s:%d Error opening file %s for writing.\n",__FILE__,__LINE__,fname); + return err; + } + + slong nelt=size; + slong out[2][1],buf[2][1]; + comm_scan(out,c,gs_long,gs_add,&nelt,1,buf); + slong start=out[0][0]; + slong nelgt=out[1][0]; + + uint write_size=nelt*sizeof(double); + if(rank==0) + write_size+=sizeof(long); // nelgt + + char *pbuf,*pbuf0; + pbuf=pbuf0=(char*)calloc(write_size,sizeof(char)); + + if(rank==0){ + write_T(pbuf0,nelgt,long,1); + } + + uint i; + for(i=0; ic); + + free(pbuf); + + return err; +} + +#undef write_T diff --git a/src/genmap-handle.c b/src/genmap-handle.c index 42eeab5f..854b2ed7 100644 --- a/src/genmap-handle.c +++ b/src/genmap-handle.c @@ -1,47 +1,42 @@ #include "genmap-impl.h" -GenmapElements GenmapGetElements(GenmapHandle h) { - return (GenmapElements) h->elementArray.ptr; +GenmapElements GenmapGetElements(genmap_handle h) { + return (GenmapElements) h->elements->ptr; } -void GenmapSetElements(GenmapHandle h, GenmapElements elements) { - h->elementArray.ptr = elements; +GenmapInt GenmapGetNLocalElements(genmap_handle h) { + return h->elements->n; } -GenmapInt GenmapGetNLocalElements(GenmapHandle h) { - return h->elementArray.n; +void GenmapSetArrayElements(genmap_handle h, struct array *localElements) { + h->elements = localElements; } -void GenmapSetNLocalElements(GenmapHandle h, GenmapInt localElements) { - array_init(struct GenmapElement_private, &h->elementArray, localElements); - h->elementArray.n = localElements; -} - -GenmapLong GenmapGetNGlobalElements(GenmapHandle h) { +GenmapLong GenmapGetNGlobalElements(genmap_handle h) { return h->nel; } -void GenmapSetNGlobalElements(GenmapHandle h, GenmapLong globalElements) { +void GenmapSetNGlobalElements(genmap_handle h, GenmapLong globalElements) { h->nel = globalElements; } -GenmapLong GenmapGetLocalStartIndex(GenmapHandle h) { +GenmapLong GenmapGetLocalStartIndex(genmap_handle h) { return h->start; } -void GenmapSetLocalStartIndex(GenmapHandle h, GenmapLong localStart) { +void GenmapSetLocalStartIndex(genmap_handle h, GenmapLong localStart) { h->start = localStart; } -int GenmapGetNVertices(GenmapHandle h) { +int GenmapGetNVertices(genmap_handle h) { return h->nv; } -void GenmapSetNVertices(GenmapHandle h, int nVertices) { +void GenmapSetNVertices(genmap_handle h, int nVertices) { h->nv = nVertices; } -void GenmapScan(GenmapHandle h, GenmapComm c) { +void GenmapScan(genmap_handle h, GenmapComm c) { GenmapLong out[2][1], buf[2][1]; GenmapLong lelt = GenmapGetNLocalElements(h); comm_scan(out,&(c->gsc),gs_long_long,gs_add,&lelt,1,buf); diff --git a/src/genmap-impl.h b/src/genmap-impl.h index 73a7b866..cdd688fb 100644 --- a/src/genmap-impl.h +++ b/src/genmap-impl.h @@ -18,62 +18,123 @@ #define GENMAP_PROC 2 #define GENMAP_ORIGIN 3 -struct GenmapComm_private { +#define GENMAP_RCB_ELEMENT 0 +#define GENMAP_RSB_ELEMENT 1 + +#define MAXDIM 3 /* Maximum dimension of the mesh */ +#define MAXNV 8 /* Maximum number of vertices per element */ + +struct GenmapComm_private{ struct comm gsc; + /* Un-weighted Laplacian */ struct gs_data *gsh; csr_mat M; + /* Weighted Laplacian */ + struct gs_data *gsw; buffer buf; GenmapScalar *b; }; -struct GenmapElement_private { - GenmapScalar fiedler; +/* parRCB internals */ +struct rcb_element{ + unsigned char type; + GenmapInt proc; + GenmapInt origin; + GenmapInt seq; GenmapLong globalId; - GenmapLong globalId0; - GenmapLong vertices[8]; + GenmapScalar coord[MAXDIM]; +}; + +void rcb_local(struct array *a,uint start,uint end,int ndim,buffer *buf); +int rcb_level(struct comm *c,struct array *a,int ndim); +int rcb(struct comm *ci,struct array *a,int ndim); + +/* parRSB internals */ +/* rsb_element should be a superset of rsb_element */ +struct rsb_element{ + unsigned char type; GenmapInt proc; GenmapInt origin; + GenmapInt seq; + GenmapLong globalId; + GenmapScalar coord[MAXDIM]; + GenmapLong vertices[8]; + GenmapInt part; + GenmapULong globalId0; + GenmapScalar fiedler; }; int GenmapCreateElements(GenmapElements *e); int GenmapDestroyElements(GenmapElements e); -GenmapElements GenmapGetElements_default(GenmapHandle h); +GenmapElements GenmapGetElements_default(genmap_handle h); + +struct genmap_handle_private { + GenmapComm global; + GenmapComm local; + + GenmapLong nel; + GenmapLong Nnodes; + GenmapLong start; + int nv; + + GenmapScalar *weights; + struct array *elements; + struct crystal cr; + int verbose_level; + int print_stat; +}; + +int GenmapCreateHandle(genmap_handle h); +int GenmapDestroyHandle(genmap_handle h); + +struct GenmapVector_private { + GenmapInt size; + GenmapScalar *data; +}; + +#define GenmapMalloc(n, p) GenmapMallocArray ((n), sizeof(**(p)), p) +#define GenmapCalloc(n, p) GenmapCallocArray ((n), sizeof(**(p)), p) +#define GenmapRealloc(n, p) GenmapReallocArray((n), sizeof(**(p)), p) + +/* Genmap Metrics */ typedef enum{ - RSB, - RCB, + ALLREDUCE=0, + AXISLEN, BINN1, BINN2, - AXISLEN, - PARSORT, - SETPROC, - LOCALSORT, + BISECT, COMMSPLIT, - LOADBALANCE0, - LOADBALANCE1, - RCBTRANSFER, - UPDATEPROBE, - GSSETUP, - PAIRWISE, CRYSTAL, - ALLREDUCE, - NNEIGHBORS, FIEDLER, - NFIEDLER, - LANCZOS, - RQI, FMG, GSOP, - LAPLACIANSETUP, - NCONN, + GSSETUP, + GRAMMIAN, + LANCZOS, + LAPLACIANSETUP0, + LAPLACIANSETUP1, LAPLACIAN, - PROJECTPF, + LOCALSORT, + NCONN, + NNEIGHBORS, + NFIEDLER, + NDISCON, NPROJECTPF, + NRQI, + PAIRWISE, + PARSORT, PRECONSETUP, - PRECONVCYCLE, PRECONAX, - END -}metric; + PROJECTPF, + PROJECT, + RCB, + RCBTRANSFER, + RQI, + SETPROC, + UPDATEPROBE, + VCYCLE +} metric; void metric_init(); void metric_finalize(); @@ -85,56 +146,20 @@ void metric_push_level(); uint metric_get_levels(); void metric_print(struct comm *c); -struct GenmapHandle_private { - GenmapComm global; - GenmapComm local; - - GenmapLong nel; - GenmapLong Nnodes; - GenmapLong start; - int nv; - - struct array elementArray; - - struct crystal cr; - - int dbgLevel; - int printStat; -}; - -int GenmapCreateHandle(GenmapHandle h); -int GenmapDestroyHandle(GenmapHandle h); - -struct GenmapVector_private { - GenmapInt size; - GenmapScalar *data; -}; - -#define GenmapMalloc(n, p) GenmapMallocArray ((n), sizeof(**(p)), p) -#define GenmapCalloc(n, p) GenmapCallocArray ((n), sizeof(**(p)), p) -#define GenmapRealloc(n, p) GenmapReallocArray((n), sizeof(**(p)), p) - -void GenmapFiedlerMinMax(GenmapHandle h, GenmapScalar *min, - GenmapScalar *max); -void GenmapGlobalIdMinMax(GenmapHandle h, GenmapLong *min, - GenmapLong *max); -GenmapInt GenmapSetFiedlerBin(GenmapHandle h); -GenmapInt GenmapSetGlobalIdBin(GenmapHandle h); -void GenmapAssignBins(GenmapHandle h, int field, buffer *buf0); -void GenmapTransferToBins(GenmapHandle h, int field, buffer *buf0); -void GenmapBinSort(GenmapHandle h, int field, buffer *buf0); - -#define MAXDIM 3 +/* genCon */ typedef struct{ - int proc; - int orig; - int seq; - unsigned long long id; - double coord[MAXDIM]; -}elm_rcb; - -int parRCB(struct comm *ci,struct array *a,int ndim); -void rcb_local(struct array *a,uint start,uint end, - int ndim,buffer *buf); + GenmapULong sequenceId; + int nNeighbors; + GenmapULong elementId; + GenmapULong vertexId; + uint workProc; +} vertex; + +/* Components */ +sint is_disconnected(struct comm *c,struct gs_data *gsh,buffer *buf, + uint nelt,uint nv); + +/* Matrix inverse */ +void matrix_inverse(int N,double *A); #endif diff --git a/src/genmap-lanczos.c b/src/genmap-lanczos.c index 97df7bcb..b6ee4572 100644 --- a/src/genmap-lanczos.c +++ b/src/genmap-lanczos.c @@ -4,8 +4,7 @@ #include /* Orthogonalize by 1-vector (vector of all 1's) */ -int GenmapOrthogonalizebyOneVector(GenmapHandle h,GenmapComm c, - GenmapVector q1, GenmapLong n) +int GenmapOrthogonalizebyOneVector(GenmapComm c, GenmapVector q1, GenmapLong n) { GenmapInt i; GenmapScalar sum = 0.0; @@ -22,7 +21,7 @@ int GenmapOrthogonalizebyOneVector(GenmapHandle h,GenmapComm c, return 0; } -int GenmapLanczosLegendary(GenmapHandle h,GenmapComm c,GenmapVector f, +int GenmapLanczosLegendary(genmap_handle h,GenmapComm c,GenmapVector f, GenmapInt niter,GenmapVector **rr,GenmapVector diag,GenmapVector upper) { assert(diag->size == niter); @@ -38,21 +37,19 @@ int GenmapLanczosLegendary(GenmapHandle h,GenmapComm c,GenmapVector f, GenmapScalar eps = 1.e-5; GenmapScalar alpha, beta; GenmapScalar rnorm, rtol, rni, rtr, rtz1, rtz2, pap = 0.0, pap_old; - GenmapVector r, p, w, weights; + GenmapVector r, p, w; rtz1 = 1.0; GenmapScalar tmp; GenmapInt lelt = GenmapGetNLocalElements(h); - GenmapCreateVector(&weights, lelt); GenmapCreateZerosVector(&p, lelt); GenmapCreateVector(&w, lelt); - GenmapInitLaplacianWeighted(h, c, weights); GenmapCreateVector(&r, lelt); GenmapCopyVector(r, f); - GenmapOrthogonalizebyOneVector(h, c, r, GenmapGetNGlobalElements(h)); + GenmapOrthogonalizebyOneVector(c, r, GenmapGetNGlobalElements(h)); rtr = GenmapDotVector(r, r); GenmapGop(c, &rtr, 1, GENMAP_SCALAR, GENMAP_SUM); rnorm = sqrt(rtr); @@ -76,9 +73,9 @@ int GenmapLanczosLegendary(GenmapHandle h,GenmapComm c,GenmapVector f, if(iter == 0) beta = 0.0; GenmapAxpbyVector(p, p, beta, r, 1.0); - GenmapOrthogonalizebyOneVector(h, c, p, GenmapGetNGlobalElements(h)); + GenmapOrthogonalizebyOneVector(c, p, GenmapGetNGlobalElements(h)); - GenmapLaplacianWeighted(h, c, p, weights, w); + GenmapLaplacianWeighted(h, c, p->data, w->data); GenmapScaleVector(w, w, -1.0); pap_old = pap; @@ -115,7 +112,6 @@ int GenmapLanczosLegendary(GenmapHandle h,GenmapComm c,GenmapVector f, } } - GenmapDestroyVector(weights); GenmapDestroyVector(p); GenmapDestroyVector(w); GenmapDestroyVector(r); @@ -123,7 +119,7 @@ int GenmapLanczosLegendary(GenmapHandle h,GenmapComm c,GenmapVector f, return iter; } -int GenmapLanczos(GenmapHandle h, GenmapComm c, GenmapVector init, +int GenmapLanczos(genmap_handle h, GenmapComm c, GenmapVector init, GenmapInt iter, GenmapVector **q, GenmapVector alpha, GenmapVector beta) { assert(alpha->size == iter); @@ -143,7 +139,7 @@ int GenmapLanczos(GenmapHandle h, GenmapComm c, GenmapVector init, GenmapCreateVector(&q1, lelt); GenmapCopyVector(q1, init); - GenmapOrthogonalizebyOneVector(h, c, q1, GenmapGetNGlobalElements(h)); + GenmapOrthogonalizebyOneVector(c, q1, GenmapGetNGlobalElements(h)); normq1 = GenmapDotVector(q1, q1); GenmapGop(c, &normq1, 1, GENMAP_SCALAR, GENMAP_SUM); normq1 = sqrt(normq1); @@ -162,16 +158,12 @@ int GenmapLanczos(GenmapHandle h, GenmapComm c, GenmapVector init, (*q)[i] = NULL; } - GenmapVector weights; - GenmapCreateVector(&weights, lelt); - GenmapInitLaplacianWeighted(h, c, weights); - int k; for(k = 0; k < iter; k++) { GenmapCreateVector(&(*q)[k], lelt); GenmapCopyVector((*q)[k], q1); - GenmapLaplacianWeighted(h, c, q1, weights, u); + GenmapLaplacianWeighted(h, c, q1->data, u->data); alpha->data[k] = GenmapDotVector(q1, u); GenmapGop(c, &alpha->data[k], 1, GENMAP_SCALAR, GENMAP_SUM); @@ -200,7 +192,6 @@ int GenmapLanczos(GenmapHandle h, GenmapComm c, GenmapVector init, GenmapDestroyVector(q0); GenmapDestroyVector(q1); GenmapDestroyVector(u); - GenmapDestroyVector(weights); return k; } diff --git a/src/genmap-laplacian-weighted.c b/src/genmap-laplacian-weighted.c index 767faf6b..8619dc3c 100644 --- a/src/genmap-laplacian-weighted.c +++ b/src/genmap-laplacian-weighted.c @@ -1,8 +1,10 @@ #include -int GenmapInitLaplacianWeighted(GenmapHandle h, GenmapComm c, GenmapVector weights) { +int GenmapInitLaplacianWeighted(genmap_handle h, GenmapComm c){ GenmapInt lelt = GenmapGetNLocalElements(h); GenmapInt nv = GenmapGetNVertices(h); + + GenmapRealloc(lelt, &h->weights); GenmapUInt numPoints = (GenmapUInt) nv * lelt; GenmapLong *vertices; @@ -11,13 +13,12 @@ int GenmapInitLaplacianWeighted(GenmapHandle h, GenmapComm c, GenmapVector weigh GenmapElements elements = GenmapGetElements(h); GenmapInt i, j; for(i = 0; i < lelt; i++) { - for(j = 0; j < nv; j++) { + for(j = 0; j < nv; j++) vertices[i * nv + j] = elements[i].vertices[j]; - } } - if(c->gsh) - gs_free(c->gsh); + if(c->gsw) + gs_free(c->gsw); #if defined(GENMAP_DEBUG) double t1 = GenmapGetMaxRss(); @@ -25,8 +26,8 @@ int GenmapInitLaplacianWeighted(GenmapHandle h, GenmapComm c, GenmapVector weigh printf("RSS before gs_setup: %lf\n", t1); #endif - c->gsh = gs_setup(vertices, numPoints, &c->gsc, 0, - gs_crystal_router, 0); + c->gsw=gs_setup(vertices,numPoints,&c->gsc,0,gs_crystal_router,0); + #if defined(GENMAP_DEBUG) t1 = GenmapGetMaxRss(); if(GenmapCommRank(GenmapGetLocalComm(h)) == 0) @@ -40,20 +41,16 @@ int GenmapInitLaplacianWeighted(GenmapHandle h, GenmapComm c, GenmapVector weigh for(j = 0; j < nv; j++) u[nv * i + j] = 1.; - gs(u, genmap_gs_scalar, gs_add, 0, c->gsh, &c->buf); - - assert(weights->size == lelt); + gs(u, gs_double, gs_add, 0, c->gsw, &c->buf); for(i = 0; i < lelt; i++) { - weights->data[i] = 0.; - for(j = 0; j < nv; j++) { - weights->data[i] += u[nv * i + j]; - } + h->weights[i] = 0.0; + for(j = 0; j < nv; j++) + h->weights[i] += u[nv * i + j]; } - for(i = 0; i < lelt; i++) { - weights->data[i] *= -1; - } + for(i = 0; i < lelt; i++) + h->weights[i] *= -1; GenmapFree(u); GenmapFree(vertices); @@ -61,11 +58,7 @@ int GenmapInitLaplacianWeighted(GenmapHandle h, GenmapComm c, GenmapVector weigh return 0; } -int GenmapLaplacianWeighted(GenmapHandle h, GenmapComm c, GenmapVector u, - GenmapVector weights, GenmapVector v) { - assert(u->size == v->size); - assert(u->size == GenmapGetNLocalElements(h)); - +int GenmapLaplacianWeighted(genmap_handle h, GenmapComm c, GenmapScalar *u, GenmapScalar *v) { GenmapInt lelt = GenmapGetNLocalElements(h); GenmapInt nv = GenmapGetNVertices(h); @@ -75,15 +68,14 @@ int GenmapLaplacianWeighted(GenmapHandle h, GenmapComm c, GenmapVector u, GenmapInt i, j; for(i = 0; i < lelt; i++) for(j = 0; j < nv; j++) - ucv[nv * i + j] = u->data[i]; + ucv[nv * i + j] = u[i]; - gs(ucv, genmap_gs_scalar, gs_add, 0, c->gsh, &c->buf); + gs(ucv, gs_double, gs_add, 0, c->gsw, &c->buf); for(i = 0; i < lelt; i++) { - v->data[i] = weights->data[i] * u->data[i]; - for(j = 0; j < nv; j ++) { - v->data[i] += ucv[nv * i + j]; - } + v[i] = h->weights[i] * u[i]; + for(j = 0; j < nv; j ++) + v[i] += ucv[nv * i + j]; } GenmapFree(ucv); diff --git a/src/genmap-laplacian.c b/src/genmap-laplacian.c index e59e7004..d7e4fd94 100644 --- a/src/genmap-laplacian.c +++ b/src/genmap-laplacian.c @@ -2,20 +2,7 @@ #define min(a,b) ((b)<(a)?(b):(a)) -typedef struct{ - GenmapULong sequenceId; - GenmapLong neighbors[8]; - int nNeighbors; - GenmapULong elementId; - GenmapULong vertexId; - uint workProc; -} vertex; - -typedef struct{ - GenmapULong elementId; -} element; - -struct array *GenmapFindNeighbors(GenmapHandle h,GenmapComm c) +struct array *GenmapFindNeighbors(genmap_handle h,GenmapComm c) { struct comm cc=c->gsc; @@ -103,7 +90,7 @@ struct array *GenmapFindNeighbors(GenmapHandle h,GenmapComm c) return nbrs; } -int GenmapInitLaplacian(GenmapHandle h,GenmapComm c) +int GenmapInitLaplacian(genmap_handle h,GenmapComm c) { struct array *entries=GenmapFindNeighbors(h,c); csr_mat_setup(entries,&c->gsc,&c->M); @@ -112,15 +99,26 @@ int GenmapInitLaplacian(GenmapHandle h,GenmapComm c) c->gsh=get_csr_top(c->M,&c->gsc); GenmapMalloc(c->M->row_off[c->M->rn],&c->b); +#if defined(GENMAP_DEBUG) + int nnz = c->M->row_off[c->M->rn]; + double fro[2] = {0.0, 0.0}, buf[2]; + for(int i=0; iM->v[i]; + fro[1] += c->M->v[i]*c->M->v[i]; + } + comm_allreduce(&c->gsc, gs_double, gs_add, &fro, 2, &buf); + printf("nrom(G,'1')=%g\nnorm(G,'fro')=%g\n", fro[0], fro[1]); + + csr_mat_print(c->M, &c->gsc); +#endif + return 0; } -int GenmapLaplacian(GenmapHandle h,GenmapComm c,GenmapVector u, - GenmapVector v) +int GenmapLaplacian(genmap_handle h,GenmapComm c,GenmapScalar *u, GenmapScalar *v) { - assert(u->size==v->size); - csr_mat_gather(c->M,c->gsh,u->data,c->b,&c->buf); - csr_mat_apply(v->data,c->M,c->b); + csr_mat_gather(c->M,c->gsh,u,c->b,&c->buf); + csr_mat_apply(v,c->M,c->b); return 0; } diff --git a/src/genmap-load-balance.c b/src/genmap-load-balance.c new file mode 100644 index 00000000..0b2d0014 --- /dev/null +++ b/src/genmap-load-balance.c @@ -0,0 +1,70 @@ +#include + +/* Load balance input data */ +void genmap_load_balance(struct array *eList,double *coord,long long *vtx, + uint nel,int nv,struct crystal *cr,buffer *bfr) { + slong in = nel; + slong out[2][1], buf[2][1]; + comm_scan(out, &cr->comm, gs_long, gs_add, &in, 1, buf); + slong nelg_start = out[0][0]; + slong nelg = out[1][0]; + + int size = cr->comm.np; + uint nstar = nelg/size; + if(nstar == 0) + nstar = 1; + + size_t unit_size; + struct rcb_element *element = NULL; + + if(vtx == NULL) { // RCB + unit_size = sizeof(struct rcb_element); + element = calloc(1, sizeof(struct rcb_element)); + element->type=GENMAP_RCB_ELEMENT; + } else { + unit_size = sizeof(struct rsb_element); + element = calloc(1, sizeof(struct rsb_element)); + element->type=GENMAP_RSB_ELEMENT; + } + + element->origin = cr->comm.id; + + array_init_(eList, nel, unit_size, __FILE__, __LINE__); + + int ndim=(nv==8)?3:2; + int e,n,v; + for(e=0; eglobalId = nelg_start + e + 1; + element->proc = (int)((eg-1)/nstar); + if(eg > size*nstar) + element->proc = (eg%size)-1; + + element->coord[0] = element->coord[1] = element->coord[2] = 0.0; + for(v=0;vcoord[n] += coord[e*ndim*nv + v*ndim + n]; + } + for(n=0;ncoord[n] /= nv; + + array_cat_(unit_size, eList, element, 1, __FILE__, __LINE__); + } + assert(eList->n==nel); + + if(vtx != NULL) { + struct rsb_element *elements = eList->ptr; + for(e=0; en; + if(vtx != NULL) + sarray_sort(struct rsb_element, eList->ptr, nel, globalId, 1, bfr); + else + sarray_sort(struct rcb_element, eList->ptr, nel, globalId, 1, bfr); + + free(element); +} diff --git a/src/genmap-matrix-inverse.c b/src/genmap-matrix-inverse.c new file mode 100644 index 00000000..9eae7a87 --- /dev/null +++ b/src/genmap-matrix-inverse.c @@ -0,0 +1,40 @@ +#include + +#if defined(GENMAP_UNDERSCORE) +# define FNAME(x) TOKEN_PASTE(x,_) +#else +# define FNAME(x) x +#endif + +#if defined(GENMAP_BLAS) + +#define FDGETRF FNAME(dgetrf) +void dgetrf_(int* M, int *N, double* A, int* lda, int* IPIV, int* INFO); +#define FDGETRI FNAME(dgetri) +void dgetri_(int* N, double* A, int* lda, int* IPIV, double* WORK, int* lwork, int* INFO); + +void matrix_inverse(int N,double *A){ + int size=N*N; + int info; + + int *ipiv=(int*) calloc(N,sizeof(int)); + double *work=(double *) calloc(N*N,sizeof(double)); + + FDGETRF(&N,&N,A,&N,ipiv,&info); + if(info!=0) + printf("dgetrf: %d\n",info); + + FDGETRI(&N,A,&N,ipiv,work,&size,&info); + if(info!=0) + printf("dgetri: %d\n",info); + + free(ipiv); + free(work); +} + +#undef FDGETRF +#undef FDGETRI + +#endif // GENMAP_BLAS + +#undef FNAME diff --git a/src/genmap-rcb.c b/src/genmap-rcb.c index 03c61393..ce8d167c 100644 --- a/src/genmap-rcb.c +++ b/src/genmap-rcb.c @@ -3,66 +3,113 @@ #include #include -void get_axis_len(double *length,struct array *a,struct comm *c,int ndim) +void get_axis_len_local(double *min,double *max,void *elems,uint nel, + int ndim) { - double min[MAXDIM],max[MAXDIM]; sint i; - for(i=0;in; - elm_rcb* elems=a->ptr; + struct rcb_element *elem; for(i=0;imax[0]) max[0]=elems[i].coord[0]; - if(elems[i].coord[1]>max[1]) max[1]=elems[i].coord[1]; - if(ndim==3) - if(elems[i].coord[2]>max[2]) max[2]=elems[i].coord[2]; + elem=(struct rcb_element*)((void*)elems+i*unit_size); + if(elem->coord[0]coord[0]; + if(elem->coord[0]>max[0]) max[0]=elem->coord[0]; + + if(elem->coord[1]coord[1]; + if(elem->coord[1]>max[1]) max[1]=elem->coord[1]; } + if(ndim==3){ + for(i=0;icoord[2]coord[2]; + if(elem->coord[2]>max[2]) max[2]=elem->coord[2]; + } + } +} + +void get_axis_len(double *length,struct array *a,struct comm *c,int ndim) +{ + double min[MAXDIM],max[MAXDIM],buf[MAXDIM]; - double buf[MAXDIM]; + get_axis_len_local(min,max,a->ptr,a->n,ndim); comm_allreduce(c,gs_double,gs_min,min,MAXDIM,buf); comm_allreduce(c,gs_double,gs_max,max,MAXDIM,buf); + sint i; for(i=0;i1){ - metric_tic(&c,AXISLEN); - get_axis_len(length,a,&c,ndim); - metric_toc(&c,AXISLEN); + if(c->np==1){ + buffer bfr; buffer_init(&bfr,1024); + rcb_local(a,0,a->n,ndim,&bfr); + buffer_free(&bfr); + return 0; + } + + metric_tic(c,AXISLEN); + get_axis_len(length,a,c,ndim); + metric_toc(c,AXISLEN); - int axis1=0,d; - for(d=1;dlength[axis1]) axis1=d; + int axis1=0,d; + for(d=1;dlength[axis1]) axis1=d; - metric_tic(&c,PARSORT); + metric_tic(c,PARSORT); + unsigned char *type=a->ptr; + if(*type==GENMAP_RCB_ELEMENT){ switch(axis1){ case 0: - parallel_sort(elm_rcb,a,coord[0],gs_double,0,1,&c); + parallel_sort(struct rcb_element,a,coord[0],gs_double,0,1,c); break; case 1: - parallel_sort(elm_rcb,a,coord[1],gs_double,0,1,&c); + parallel_sort(struct rcb_element,a,coord[1],gs_double,0,1,c); break; case 2: - parallel_sort(elm_rcb,a,coord[2],gs_double,0,1,&c); + parallel_sort(struct rcb_element,a,coord[2],gs_double,0,1,c); break; default: break; } - metric_toc(&c,PARSORT); + }else if(*type==GENMAP_RSB_ELEMENT){ + switch(axis1){ + case 0: + parallel_sort(struct rsb_element,a,coord[0],gs_double,0,1,c); + break; + case 1: + parallel_sort(struct rsb_element,a,coord[1],gs_double,0,1,c); + break; + case 2: + parallel_sort(struct rsb_element,a,coord[2],gs_double,0,1,c); + break; + default: + break; + } + } + metric_toc(c,PARSORT); + + return 0; +} + +int rcb(struct comm *ci,struct array *a,int ndim){ + struct comm c; comm_dup(&c,ci); + sint rank=c.id; + sint size=c.np; + while(size>1){ + rcb_level(&c,a,ndim); int p=(size+1)/2; int bin=(rank>=p); @@ -75,8 +122,6 @@ int parRCB(struct comm *ci,struct array *a,int ndim){ #endif metric_toc(&c,COMMSPLIT); - metric_push_level(); - rank=c.id; size=c.np; } @@ -85,54 +130,61 @@ int parRCB(struct comm *ci,struct array *a,int ndim){ return 0; } -void get_axis_len_local(double *length,elm_rcb *elems,uint nel,int ndim) -{ - double min[MAXDIM],max[MAXDIM]; - sint i; - for(i=0;i=0); - for(i=0;imax[0]) max[0]=elems[i].coord[0]; - if(elems[i].coord[1]>max[1]) max[1]=elems[i].coord[1]; - if(ndim==3) - if(elems[i].coord[2]>max[2]) max[2]=elems[i].coord[2]; + if(size<=2) return; + + size_t unit_size; + unsigned char *type=a->ptr; + if(*type==GENMAP_RCB_ELEMENT){ + unit_size=sizeof(struct rcb_element); + }else if(*type==GENMAP_RSB_ELEMENT){ + unit_size=sizeof(struct rsb_element); } + void *st=(void *)a->ptr+unit_size*start; + double length[3],min[3],max[3]; + get_axis_len_local(min,max,st,size,ndim); + + sint i; for(i=0;i=start); - - uint size=end-start; - if(size<=2) return; - - double length[3]; elm_rcb *st=a->ptr; st+=start; - get_axis_len_local(length,st,size,ndim); int axis=0; if(fabs(length[axis]) #include +#include +#include -void GenmapRSB(GenmapHandle h,int verbose){ - int maxIter=50; - int npass =50; +void genmap_rsb(genmap_handle h,int verbose){ + int max_iter=50; + int max_pass=50; - GenmapInt i; - GenmapElements e = GenmapGetElements(h); - GenmapScan(h, GenmapGetLocalComm(h)); - for(i = 0; i < GenmapGetNLocalElements(h); i++) { - e[i].globalId =GenmapGetLocalStartIndex(h)+i+1; - e[i].globalId0=GenmapGetLocalStartIndex(h)+i+1; - } + GenmapComm local_c=GenmapGetLocalComm(h); + struct comm *lc=&local_c->gsc; GenmapComm global_c=GenmapGetGlobalComm(h); - GenmapComm local_c =GenmapGetLocalComm(h); - int rank=GenmapCommRank(global_c); + struct comm *gc=&global_c->gsc; + + GenmapScan(h,local_c); + crystal_init(&h->cr,lc); - if(rank == 0 && h->dbgLevel > 0) - printf("running RSB "); - fflush(stdout); + GenmapScan(h, GenmapGetLocalComm(h)); + uint nelt=GenmapGetNLocalElements(h); + GenmapElements e=GenmapGetElements(h); + GenmapInt i; + for(i=0; icr), &(h->local->gsc)); - buffer buf0 = null_buffer; + buffer buf; buffer_init(&buf,1024); - int nve =h->nv; + int nve=h->nv; int ndim=(nve==8)?3:2; int level=0; + int np; - metric_init(); // init metrics - - while(GenmapCommSize(GenmapGetLocalComm(h)) > 1){ - GenmapComm local_c=GenmapGetLocalComm(h); - struct comm *gsc=&local_c->gsc; - GenmapInt np=gsc->np; - - metric_tic(gsc,RSB); + while(GenmapCommSize(GenmapGetLocalComm(h))>1){ + local_c=GenmapGetLocalComm(h); + lc=&local_c->gsc; + np=lc->np; #if defined(GENMAP_PAUL) int global=1; #else - int global=(np==GenmapCommSize(GenmapGetGlobalComm(h))); + int global=(np==gc->np); +#endif + +#if defined(GENMAP_RCB_PRE_STEP) + /* Run RCB pre-step */ + metric_tic(lc,RCB); + rcb(lc,h->elements,ndim); + rcb_local(h->elements,0,h->elements->n,ndim,&buf); + metric_toc(lc,RCB); +#else + /* Sort by global id otherwise */ + parallel_sort(struct rsb_element,h->elements,globalId0,gs_long,0,1,lc); #endif + /* Initialize the laplacian */ + metric_tic(lc,LAPLACIANSETUP0); + GenmapInitLaplacianWeighted(h,local_c); + metric_toc(lc,LAPLACIANSETUP0); + + /* Run fiedler */ + metric_tic(lc,FIEDLER); int ipass=0,iter; - do{ - metric_tic(gsc,FIEDLER); + do { #if defined(GENMAP_LANCZOS) - iter=GenmapFiedlerLanczos(h,local_c,maxIter,global); + iter=GenmapFiedlerLanczos(h,local_c,max_iter,global); #elif defined(GENMAP_RQI) - iter=GenmapFiedlerRQI(h,local_c,maxIter,global); + iter=GenmapFiedlerRQI(h,local_c,max_iter,global); #endif - metric_toc(gsc,FIEDLER); metric_acc(NFIEDLER,iter); - global=0; - }while(++ipasselements,fiedler,gs_double,0,1,lc); + int bin=1; + if(lc->id<(np+1)/2) + bin=0; + // FIXME: Ugly GenmapSplitComm(h,&local_c,bin); GenmapSetLocalComm(h,local_c); + lc=&local_c->gsc; + GenmapScan(h,local_c); + metric_toc(lc,BISECT); -#if defined(GENMAP_PAUL) - GenmapBinSort(h,GENMAP_GLOBALID,&buf0); -#endif - - level++; metric_push_level(); + level++; } - sint converged=1,buf; + /* Check if Fidler converged */ + sint converged=1; for(i=0; i=npass*maxIter){ - converged=0; - break; + if(val>=max_pass*max_iter){ + converged=0; break; } } - comm_allreduce(&global_c->gsc,gs_int,gs_min,&converged,1,&buf);// min - if(converged==0 && GenmapCommRank(global_c)==0){ - printf("WARNING: Lanczos failed to converge during partitioning!"); - } + sint bfr; + comm_allreduce(gc,gs_int,gs_min,&converged,1,&bfr);// min + if(converged==0 && gc->id==0) + printf("\tWARNING: Lanczos failed to converge while partitioning!\n"); + + /* Check for disconnected components */ + GenmapInitLaplacianWeighted(h,global_c); - //metric_print(&global_c->gsc); - metric_finalize(); + sint discon=is_disconnected(gc,global_c->gsw,&global_c->buf,nelt,nve); + if(discon>0 && gc->id==0) + printf("\tWarning: There are disconnected components!\n"); - crystal_free(&(h->cr)); - buffer_free(&buf0); + crystal_free(&h->cr); + buffer_free(&buf); } diff --git a/src/genmap-statistics.c b/src/genmap-statistics.c index b071eff3..00be71ba 100644 --- a/src/genmap-statistics.c +++ b/src/genmap-statistics.c @@ -5,19 +5,19 @@ #include -#define MAXMETS 150 -#define MAXLVLS 30 +#define MAXMETS 50 +#define MAXLVLS 30 #define MAXSIZE (MAXMETS*MAXLVLS) static double metrics[MAXMETS]; static double *stack; -static uint stack_size,stack_max; +static uint stack_size; void metric_init(){ uint i; for(i=0; i stack_max"); - - if(stack_size==stack_max){ - stack_max+=stack_size/2+1; - GenmapRealloc(stack_max*MAXMETS,&stack); - } + assert(stack_size= MAXLVLS"); uint i; for(i=0; inp; @@ -79,23 +81,32 @@ void metric_print(struct comm *c){ for(i=0; iid==0){ - printf("level=%02d\n",i); - printf(" BINN1 : %g/%g/%g\n",SUMMARY(i,BINN1 )); - printf(" BINN2 : %g/%g/%g\n",SUMMARY(i,BINN2 )); - printf(" AXISLEN : %g/%g/%g\n",SUMMARY(i,AXISLEN )); - printf(" LOCALSORT : %g/%g/%g\n",SUMMARY(i,LOCALSORT )); - printf(" SETPROC : %g/%g/%g\n",SUMMARY(i,SETPROC )); - printf(" RCBTRANSFER : %g/%g/%g\n",SUMMARY(i,RCBTRANSFER )); - printf(" COMMSPLIT : %g/%g/%g\n",SUMMARY(i,COMMSPLIT)); - printf(" LOADBALANCE0 : %g/%g/%g\n",SUMMARY(i,LOADBALANCE0)); - printf(" LOADBALANCE1 : %g/%g/%g\n",SUMMARY(i,LOADBALANCE1)); - printf(" PARSORT : %g/%g/%g\n",SUMMARY(i,PARSORT )); - printf(" UPDATEPROBE : %g/%g/%g\n",SUMMARY(i,UPDATEPROBE )); + printf("level=%02d\n",i); + printf(" RCB : %g/%g/%g\n",SUMMARY(i,RCB)); + printf(" LAPLACIANSETUP0 : %g/%g/%g\n",SUMMARY(i,LAPLACIANSETUP0)); + printf(" FIEDLER : %g/%g/%g\n",SUMMARY(i,FIEDLER)); + printf(" NFIEDLER : %g/%g/%g\n",SUMMARY(i,NFIEDLER)); + printf(" LAPLACIANSETUP1 : %g/%g/%g\n",SUMMARY(i,LAPLACIANSETUP1)); + printf(" PRECONSETUP : %g/%g/%g\n",SUMMARY(i,PRECONSETUP)); + printf(" RQI : %g/%g/%g\n",SUMMARY(i,RQI)); + printf(" NRQI : %g/%g/%g\n",SUMMARY(i,NRQI)); + printf(" PROJECTPF : %g/%g/%g\n",SUMMARY(i,PROJECTPF)); + printf(" NPROJECTPF : %g/%g/%g\n",SUMMARY(i,NPROJECTPF)); + printf(" VCYCLE : %g/%g/%g\n",SUMMARY(i,VCYCLE)); + printf(" LAPLACIAN : %g/%g/%g\n",SUMMARY(i,LAPLACIAN)); + printf(" PROJECT : %g/%g/%g\n",SUMMARY(i,PROJECT)); + printf(" GRAMMIAN : %g/%g/%g\n",SUMMARY(i,GRAMMIAN)); + printf(" BISECT : %g/%g/%g\n",SUMMARY(i,BISECT)); } } -} + + GenmapFree(min); + GenmapFree(max); + GenmapFree(sum); + GenmapFree(buf); #undef SUMMARY +} #undef MAXMETS #undef MAXLVLS diff --git a/src/genmap.c b/src/genmap.c index 98dc03ef..cfb09b99 100644 --- a/src/genmap.c +++ b/src/genmap.c @@ -3,30 +3,36 @@ #include -int GenmapInit(GenmapHandle *h, GenmapCommExternal ce) { - GenmapMalloc(1, h); - GenmapHandle h_ = *h; +int genmap_init(genmap_handle *h_, comm_ext ce, int *options) { + GenmapMalloc(1, h_); + genmap_handle h = *h_; - GenmapCreateComm(&h_->global, ce); - GenmapCreateComm(&h_->local, ce); + GenmapCreateComm(&h->global, ce); + GenmapCreateComm(&h->local, ce); - h_->elementArray.ptr = NULL; - h_->elementArray.n = (*h)->elementArray.max = 0; + h->weights = NULL; - h_->dbgLevel = 0; - h_->printStat = 0; + h->verbose_level = 0; + h->print_stat = 0; + + if(options!=NULL){ + if(options[0]>0) + h->verbose_level=options[1]; + if(options[0]>1) + h->print_stat=options[2]; + } return 0; } -int GenmapFinalize(GenmapHandle h) { +int genmap_finalize(genmap_handle h) { + if(h->weights!=NULL) + GenmapFree(h->weights); if(GenmapGetGlobalComm(h)) GenmapDestroyComm(GenmapGetGlobalComm(h)); if(GenmapGetLocalComm(h)) GenmapDestroyComm(h->local); - array_free(&(h->elementArray)); - GenmapFree(h); return 0; diff --git a/src/genmap.h b/src/genmap.h index d6ba73a5..eabffe11 100644 --- a/src/genmap.h +++ b/src/genmap.h @@ -20,45 +20,42 @@ #define GENMAP_MAX_READERS 32 typedef MPI_Datatype GenmapDataType; -typedef MPI_Comm GenmapCommExternal; - typedef struct GenmapComm_private *GenmapComm; -typedef struct GenmapHandle_private *GenmapHandle; +typedef struct genmap_handle_private *genmap_handle; typedef struct GenmapVector_private *GenmapVector; -typedef struct GenmapElement_private *GenmapElements; +typedef struct rsb_element *GenmapElements; -int GenmapInit(GenmapHandle *h, GenmapCommExternal ce); -int GenmapFinalize(GenmapHandle h); +int genmap_init(genmap_handle *h, comm_ext ce, int *options); +int genmap_finalize(genmap_handle h); int GenmapMallocArray(size_t n, size_t unit, void *p); int GenmapCallocArray(size_t n, size_t unit, void *p); int GenmapReallocArray(size_t n, size_t unit, void *p); int GenmapFree(void *p); -GenmapElements GenmapGetElements(GenmapHandle h); -void GenmapSetElements(GenmapHandle h, GenmapElements elements); +GenmapElements GenmapGetElements(genmap_handle h); -GenmapComm GenmapGetLocalComm(GenmapHandle h); -void GenmapSetLocalComm(GenmapHandle h, GenmapComm c); +GenmapComm GenmapGetLocalComm(genmap_handle h); +void GenmapSetLocalComm(genmap_handle h, GenmapComm c); -GenmapComm GenmapGetGlobalComm(GenmapHandle h); -void GenmapSetGlobalComm(GenmapHandle h, GenmapComm c); +GenmapComm GenmapGetGlobalComm(genmap_handle h); +void GenmapSetGlobalComm(genmap_handle h, GenmapComm c); -GenmapInt GenmapGetNLocalElements(GenmapHandle h); -void GenmapSetNLocalElements(GenmapHandle h, GenmapInt localElements); +GenmapInt GenmapGetNLocalElements(genmap_handle h); +void GenmapSetArrayElements(genmap_handle h, struct array *localElements); -GenmapLong GenmapGetNGlobalElements(GenmapHandle h); -void GenmapSetNGlobalElements(GenmapHandle h, GenmapLong globalElements); +GenmapLong GenmapGetNGlobalElements(genmap_handle h); +void GenmapSetNGlobalElements(genmap_handle h, GenmapLong globalElements); -GenmapLong GenmapGetLocalStartIndex(GenmapHandle h); -void GenmapSetLocalStartIndex(GenmapHandle h, GenmapLong localStart); +GenmapLong GenmapGetLocalStartIndex(genmap_handle h); +void GenmapSetLocalStartIndex(genmap_handle h, GenmapLong localStart); -int GenmapGetNVertices(GenmapHandle h); -void GenmapSetNVertices(GenmapHandle, int nVertices); +int GenmapGetNVertices(genmap_handle h); +void GenmapSetNVertices(genmap_handle h, int nVertices); -void GenmapScan(GenmapHandle h, GenmapComm c); +void GenmapScan(genmap_handle h, GenmapComm c); -int GenmapCreateComm(GenmapComm *c, GenmapCommExternal ce); +int GenmapCreateComm(GenmapComm *c, comm_ext ce); int GenmapCommSize(GenmapComm c); int GenmapCommRank(GenmapComm c); @@ -69,12 +66,12 @@ int GenmapReduce(GenmapComm c, void *out, void *in, GenmapInt size, int GenmapBcast(GenmapComm c, void *in, GenmapInt count, GenmapDataType type); int GenmapDestroyComm(GenmapComm c); -void GenmapSplitComm(GenmapHandle h, GenmapComm *c, int bin); -int GenmapCrystalInit(GenmapHandle h, GenmapComm c); -int GenmapCrystalTransfer(GenmapHandle h, int field); -int GenmapCrystalFinalize(GenmapHandle h); +void GenmapSplitComm(genmap_handle h, GenmapComm *c, int bin); +int GenmapCrystalInit(genmap_handle h, GenmapComm c); +int GenmapCrystalTransfer(genmap_handle h, int field); +int GenmapCrystalFinalize(genmap_handle h); -int GenmapRead(GenmapHandle h, void *data); +int GenmapRead(genmap_handle h, void *data); int GenmapCreateVector(GenmapVector *x, GenmapInt size); int GenmapSetVector(GenmapVector x, GenmapScalar *array); @@ -97,46 +94,44 @@ GenmapScalar GenmapAbsMinVector(GenmapVector x); GenmapScalar GenmapMinVector(GenmapVector x); GenmapScalar GenmapNormVector(GenmapVector x, GenmapInt p); +int GenmapOrthogonalizebyOneVector(GenmapComm c, GenmapVector q1, GenmapLong n); + int GenmapPrintVector(GenmapVector x); int GenmapDestroyVector(GenmapVector x); /* Laplacian */ -struct array *GenmapFindNeighbors(GenmapHandle h,GenmapComm c); - -int GenmapInitLaplacianWeighted(GenmapHandle h, GenmapComm c, - GenmapVector weights); -int GenmapInitLaplacian(GenmapHandle h, GenmapComm c); +struct array *GenmapFindNeighbors(genmap_handle h,GenmapComm c); -int GenmapLaplacianWeighted(GenmapHandle h, GenmapComm c, GenmapVector u, - GenmapVector weights, GenmapVector v); -int GenmapLaplacian(GenmapHandle h, GenmapComm c, GenmapVector u, - GenmapVector v); +int GenmapInitLaplacian(genmap_handle h, GenmapComm c); +int GenmapLaplacian(genmap_handle h, GenmapComm c, GenmapScalar *u, GenmapScalar *v); +int GenmapInitLaplacianWeighted(genmap_handle h, GenmapComm c); +int GenmapLaplacianWeighted(genmap_handle h, GenmapComm c, GenmapScalar *u, GenmapScalar *v); /* Eigen */ -int GenmapInvPowerIter(GenmapVector eVector, GenmapVector alpha, - GenmapVector beta, GenmapVector init, int iter); -int GenmapTQLI(GenmapHandle h, GenmapVector diagonal, GenmapVector upper, - GenmapVector **eVectors, GenmapVector *eValues); - -int GenmapOrthogonalizebyOneVector(GenmapHandle h, GenmapComm c, - GenmapVector q1, GenmapLong n); +int GenmapInvPowerIter(GenmapVector eVector, GenmapVector alpha, GenmapVector beta, GenmapVector init, int iter); +int GenmapTQLI(genmap_handle h, GenmapVector diag, GenmapVector upper, GenmapVector **eVec, GenmapVector *eVal); +int genmap_inverse_power(double *y, int N, double *A, int verbose); +int genmap_power(double *y, int N, double *A, int verbose); /* Lanczos */ -int GenmapLanczosLegendary(GenmapHandle h, GenmapComm c, GenmapVector f, +int GenmapLanczosLegendary(genmap_handle h, GenmapComm c, GenmapVector f, GenmapInt niter, GenmapVector **rr, GenmapVector diag, GenmapVector upper); -int GenmapLanczos(GenmapHandle h, GenmapComm c, GenmapVector init, +int GenmapLanczos(genmap_handle h, GenmapComm c, GenmapVector init, GenmapInt iter, GenmapVector **q, GenmapVector alpha, GenmapVector beta); /* Fiedler */ -int GenmapFiedlerLanczos(GenmapHandle h,GenmapComm c,int maxIter, +int GenmapFiedlerLanczos(genmap_handle h,GenmapComm c,int maxIter, int global); -int GenmapFiedlerRQI(GenmapHandle h,GenmapComm c,int maxIter,int global); +int GenmapFiedlerRQI(genmap_handle h,GenmapComm c,int maxIter,int global); /* RSB */ -void GenmapRSB(GenmapHandle h,int verbose); +void genmap_rsb(genmap_handle h,int verbose); double GenmapGetMaxRss(); void GenmapPrintStack(); + +void matrix_inverse(int N,double *A); + #endif diff --git a/src/parCon.c b/src/parCon.c index 9d5d7f91..f85f00cb 100644 --- a/src/parCon.c +++ b/src/parCon.c @@ -25,15 +25,19 @@ int parRSB_findConnectivity(long long *vertexid,double *coord, struct comm c; comm_init(&c,comm); uint rank=c.id,size=c.np; - if(rank==0 && verbose) + if(rank==0 && verbose>0) printf("generating connectivity ... "); fflush(stdout); + if(verbose>1){ + printf("\tnelt/ndim/nperiodic: %d/%d/%d\n",nelt,ndim,nPeriodicFaces); + } + double t_con=0.0; comm_barrier(&c); t_con-=comm_time(); - Mesh mesh; MeshInit(&mesh,nelt,ndim); + Mesh mesh; mesh_init(&mesh,nelt,ndim); slong out[2][1],buff[2][1],in=nelt; comm_scan(out,&c,gs_long,gs_add,&in,1,buff); @@ -78,29 +82,25 @@ int parRSB_findConnectivity(long long *vertexid,double *coord, transferBoundaryFaces(mesh,&c); findMinNeighborDistance(mesh); - findSegments(mesh,&c,tol); + findSegments(mesh,&c,tol,0); setGlobalID(mesh,&c); sendBack(mesh,&c); + faceCheck(mesh,&c); matchPeriodicFaces(mesh,&c); // copy output Point ptr=mesh->elements.ptr; - k=0; - for(i=0; ielementId+1; - for(j=0; jglobalId+1; - ptr++; - } + for(i=0; i0) printf(" finished in %g s\n",t_con); - MeshFree(mesh); + mesh_free(mesh); comm_free(&c); return 0; diff --git a/src/parRCB.c b/src/parRCB.c index 7a731a86..9e2d13d0 100644 --- a/src/parRCB.c +++ b/src/parRCB.c @@ -3,135 +3,133 @@ #include #include -#include #include #include -void fparRCB_partMesh(int *part,int *seq,double *vtx,int *nel,int *nv, +void fparRCB_partMesh(int *part,int *seq,double *coord,int *nel,int *nv, int *options,int *comm,int *err) { *err = 1; - - comm_ext c; c = MPI_Comm_f2c(*comm); - *err=parRCB_partMesh(part,seq,vtx,*nel,*nv,options,c); + comm_ext c = MPI_Comm_f2c(*comm); + *err=parRCB_partMesh(part,seq,coord,*nel,*nv,options,c); } -// vtx = [nel,nv,ndim] -int parRCB_partMesh(int *part,int *seq,double *vtx,int nel,int nv, +// coord = [nel,nv,ndim] +int parRCB_partMesh(int *part,int *seq,double *coord,int nel,int nv, int *options,MPI_Comm comm) { struct comm c; comm_init(&c,comm); int rank=c.id,size=c.np; - comm_barrier(&c); - double time=comm_time(); - if(rank==0) - printf("running RCB ..."); + printf("running RCB ... "); fflush(stdout); - slong out[2][1],buf[2][1]; - slong nell=nel; - comm_scan(out,&c,gs_long,gs_add,&nell,1,buf); + comm_barrier(&c); + double time0=comm_time(); + + /* Load balance input data */ + slong out[2][1],buf[2][1],in=nel; + comm_scan(out,&c,gs_long,gs_add,&in,1,buf); slong nelg_start=out[0][0]; - slong nelg =out[1][0]; + slong nelg=out[1][0]; - struct array a; array_init(elm_rcb,&a,nel); - elm_rcb *data=a.ptr; + GenmapLong nstar=nelg/size; + if(nstar==0) nstar=1; - int ndim=(nv==8)?3:2; + struct array eList; + array_init(struct rcb_element,&eList,nel); + struct rcb_element data; + data.type=GENMAP_RCB_ELEMENT; + data.origin=rank; + + int ndim=(nv==8)?3:2; int e,n,v; for(e=0;esize*nstar) data.proc= (eg%size)-1; + + data.coord[0]=data.coord[1]=data.coord[2]=0.0; for(v=0;v0,rank,&new); - comm_init(&rcb,new); MPI_Comm_free(&new); + comm_init(&comm_rcb,new); MPI_Comm_free(&new); #else - comm_init(&rcb,1); + comm_init(&comm_rcb,1); #endif if(nel>0){ - parRCB(&rcb,&a,ndim); + metric_init(); + + rcb(&comm_rcb,&eList,ndim); // Do a local RCB if seq!=NULL if(seq!=NULL){ - uint s1=0,e1=a.n; - rcb_local(&a,s1,e1,ndim,&bfr); + rcb_local(&eList,0,eList.n,ndim,&bfr); - elm_rcb *ptr=a.ptr; - int i; - for(i=0; i #include -#define MAXNV 8 /* maximum number of vertices per element */ -typedef struct { - int proc; - GenmapLong id; - int part; - long long vtx[MAXNV]; -} elm_data; - -void fparRSB_partMesh(int *part,long long *vtx,int *nel,int *nve, - int *options,int *comm,int *err) +void fparRSB_partMesh(int *part,long long *vtx,double *coord,int *nel, + int *nve,int *options,int *comm,int *err) { *err = 1; - GenmapCommExternal c; - c = MPI_Comm_f2c(*comm); - *err = parRSB_partMesh(part, vtx, *nel, *nve, options, c); + comm_ext c = MPI_Comm_f2c(*comm); + *err = parRSB_partMesh(part, vtx, coord, *nel, *nve, options, c); } -int parRSB_partMesh(int *part,long long *vtx,int nel,int nve, - int *options,MPI_Comm comm) +/* + * part = [nel], out, + * vtx = [nel x nve], in, + * coord = [nel x nve x ndim], in, + * nel = in, + * nve = in, + * options = null or [options[0]], in */ +int parRSB_partMesh(int *part,long long *vtx,double *coord,int nel, + int nve,int *options,MPI_Comm comm) { - int rank,size; - MPI_Comm_rank(comm,&rank); - MPI_Comm_size(comm,&size); + struct comm c; comm_init(&c,comm); + int rank=c.id,size=c.np; + + if(rank==0) + printf("running RSB ..."); + fflush(stdout); + + comm_barrier(&c); + double time0=comm_time(); - /* load balance input data */ - GenmapLong nelg; - GenmapLong nell = nel; - MPI_Allreduce(&nell,&nelg,1,MPI_LONG_LONG_INT,MPI_SUM,comm); - GenmapLong nstar = nelg/size; - if(nstar == 0) nstar = 1; + /* Load balance input data */ + slong out[2][1],buf[3][1],in=nel; + comm_scan(out,&c,gs_long,gs_add,&in,1,buf); + slong nelg_start=out[0][0]; + slong nelg=out[1][0]; - GenmapLong nelg_start; - MPI_Scan(&nell,&nelg_start,1,MPI_LONG_LONG_INT,MPI_SUM,comm); - nelg_start-=nel; + GenmapLong nstar=nelg/size; + if(nstar==0) nstar=1; struct array eList; - elm_data *data; + array_init(struct rsb_element,&eList,nel); - array_init(elm_data,&eList,nel), eList.n=nel; - int e, n; - for(data=eList.ptr,e=0; esize*nstar) data[e].proc= (eg%size)-1; + int ndim=(nve==8)?3:2; + int e,n,v; + for(e=0; esize*nstar) data.proc= (eg%size)-1; + + data.coord[0]=data.coord[1]=data.coord[2]=0.0; for(n=0; n0, rank, &commRSB); + sarray_transfer(struct rsb_element,&eList,proc,1,&cr); + nel=eList.n; + struct rsb_element *e_ptr=eList.ptr; + + buffer bfr; buffer_init(&bfr,1024); + sarray_sort(struct rsb_element,eList.ptr,(unsigned)nel,globalId,1,&bfr); + + double time1=comm_time(); + comm_barrier(&c); + double time2=comm_time(); + + /* Run RSB now */ + struct comm comm_rsb; + comm_ext old=c.c; +#ifdef MPI + MPI_Comm new; MPI_Comm_split(old,nel>0,rank,&new); + comm_init(&comm_rsb,new); MPI_Comm_free(&new); +#else + comm_init(&comm_rsb,1); +#endif + + int verbose_level = 0; + int print_stat = 0; + if(options!=NULL){ + if(options[0]>0) + verbose_level=options[1]; + if(options[0]>1) + print_stat=options[2]; + } if(nel>0) { - double time0 = comm_time(); - GenmapHandle h; - GenmapInit(&h, commRSB); + metric_init(); - if(options[0] != 0) { - h->dbgLevel = options[1]; - h->printStat = options[2]; - } + genmap_handle h; + genmap_init(&h, comm_rsb.c, options); - GenmapSetNLocalElements(h, (GenmapInt)nel); + GenmapSetArrayElements(h,&eList); GenmapScan(h, GenmapGetGlobalComm(h)); GenmapSetNVertices(h, nve); - GenmapLong nelg = GenmapGetNGlobalElements(h); - GenmapInt id = GenmapCommRank(GenmapGetGlobalComm(h)); - GenmapInt size_ = GenmapCommSize(GenmapGetGlobalComm(h)); - if((GenmapLong)size_ > nelg) { - if(id == 0) + GenmapLong nelg=GenmapGetNGlobalElements(h); + GenmapInt id=GenmapCommRank(GenmapGetGlobalComm(h)); + GenmapInt size_=GenmapCommSize(GenmapGetGlobalComm(h)); + if((GenmapLong)size_>nelg){ + if(id==0) printf("Total number of elements is smaller than the " "number of processors.\n" "Run with smaller number of processors.\n"); return 1; } - GenmapElements e = GenmapGetElements(h); - GenmapLong start = GenmapGetLocalStartIndex(h); - - GenmapInt i, j; - for(i = 0; i < nel; i++) { - e[i].origin = id; - for(j = 0; j < nve; j++) { - e[i].vertices[j] = data[i].vtx[j]; - } - } - - GenmapRSB(h,h->dbgLevel>1); - - e = GenmapGetElements(h); - for(j = 0; j < GenmapGetNLocalElements(h); j++) { - e[j].proc = GenmapCommRank(GenmapGetGlobalComm(h)); - } - GenmapCrystalInit(h, GenmapGetGlobalComm(h)); - GenmapCrystalTransfer(h, GENMAP_ORIGIN); - GenmapCrystalFinalize(h); + genmap_rsb(h,h->verbose_level>1); - assert(GenmapGetNLocalElements(h) == nel); + genmap_finalize(h); - e = GenmapGetElements(h); - sarray_sort(struct GenmapElement_private,e,(unsigned int)nel,globalId, - TYPE_LONG, &buf); + if(print_stat>0) + metric_print(&c); + metric_finalize(); + } - for(i = 0; i < nel; i++) { - data[i].part = e[i].proc; + double time3=comm_time(); + comm_barrier(&c); + double time4=comm_time(); + + /* Restore original input */ + sarray_transfer(struct rsb_element,&eList,origin,1,&cr); + nel=eList.n; + sarray_sort(struct rsb_element,eList.ptr,nel,globalId,1,&bfr); + + e_ptr=eList.ptr; + for(e=0;e0){ + double min[3],max[3],sum[3]; + min[0]=max[0]=sum[0]=time1-time0; + min[1]=max[1]=sum[1]=time3-time2; + min[2]=max[2]=sum[2]=time5-time4; + comm_allreduce(&c,gs_double,gs_min,min,3,buf); // min + comm_allreduce(&c,gs_double,gs_max,max,3,buf); // max + comm_allreduce(&c,gs_double,gs_add,sum,3,buf); // sum + if(rank==0){ + printf("LOADBALANCE : %g/%g/%g\n",min[0],max[0],sum[0]/c.np); + printf("RSB : %g/%g/%g\n",min[1],max[1],sum[1]/c.np); + printf("RESTORE : %g/%g/%g\n",min[2],max[2],sum[2]/c.np); } - - if(id == 0 && h->dbgLevel > 0) - printf(" finished in %lfs\n", comm_time() - time0); - - GenmapFinalize(h); fflush(stdout); } - MPI_Comm_free(&commRSB); - - /* restore original input */ - sarray_transfer(elm_data,&eList,proc,0,&cr); - data=eList.ptr; - nel =eList.n; - sarray_sort(elm_data,data,(unsigned int)nel,id,TYPE_LONG,&buf); - MPI_Barrier(comm); - - for(e = 0; e < nel; e++) { - part[e]=data[e].part; - } - array_free(&eList); - buffer_free(&buf); + buffer_free(&bfr); crystal_free(&cr); + comm_free(&comm_rsb); comm_free(&c); return 0; diff --git a/src/parRSB.h b/src/parRSB.h index e327249a..6093d800 100644 --- a/src/parRSB.h +++ b/src/parRSB.h @@ -3,18 +3,18 @@ #define fparRSB_partMesh \ FORTRAN_UNPREFIXED(fparrsb_partmesh,FPARRSB_PARTMESH) -void fparRSB_partMesh(int *part,long long *vtx,int *nel, +void fparRSB_partMesh(int *part,long long *vtx,double *coord,int *nel, int *nve,int *options,int *comm,int *err); -int parRSB_partMesh(int *part,long long *vtx,int nel,int nve, - int *options,MPI_Comm comm); +int parRSB_partMesh(int *part,long long *vtx,double *coord,int nel, + int nve,int *options,MPI_Comm comm); #define fparRCB_partMesh \ FORTRAN_UNPREFIXED(fparrcb_partmesh,FPARRCB_PARTMESH) -void fparRCB_partMesh(int *part,int *seq,double *vtx,int *nel,int *nv, +void fparRCB_partMesh(int *part,int *seq,double *coord,int *nel,int *nv, int *options,int *comm,int *err); -int parRCB_partMesh (int *part,int *seq,double *vtx,int nel,int nv, +int parRCB_partMesh(int *part,int *seq,double *coord,int nel,int nv, int *options,MPI_Comm comm); #define fparRSB_findConnectivity\ diff --git a/src/precond/genmap-multigrid-csr.c b/src/precond/genmap-multigrid-csr.c index 714c12a7..2f51a765 100644 --- a/src/precond/genmap-multigrid-csr.c +++ b/src/precond/genmap-multigrid-csr.c @@ -138,12 +138,12 @@ void csr_mat_print(csr_mat M,struct comm *c){ for(k=0; knp; k++){ comm_barrier(c); - if(c->id==k) + if(c->id==k){ for(i=0; iid,M->row_start+i, - col[j],v[j]); + fprintf(stderr,"(%lld,%lld) -> %.10lf\n",M->row_start+i,col[j],v[j]); } + } fflush(stderr); } } diff --git a/src/precond/genmap-multigrid-flexcg.c b/src/precond/genmap-multigrid-flexcg.c index 4d4b8434..8aa3598a 100644 --- a/src/precond/genmap-multigrid-flexcg.c +++ b/src/precond/genmap-multigrid-flexcg.c @@ -4,7 +4,7 @@ #include #include -int flex_cg(GenmapHandle h,GenmapComm c,mgData d,GenmapVector ri, +int flex_cg(genmap_handle h,GenmapComm c,mgData d,GenmapVector ri, int maxIter,int verbose,GenmapVector x) { assert(x->size==ri->size); @@ -34,13 +34,9 @@ int flex_cg(GenmapHandle h,GenmapComm c,mgData d,GenmapVector ri, for(i=0; idata[i]=0.0,r->data[i]=ri->data[i]; -#if PREC - mg_vcycle(z->data,r->data,d); -#else GenmapCopyVector(z,r); -#endif #if ORTH - GenmapOrthogonalizebyOneVector(h,c,z,nelg); + GenmapOrthogonalizebyOneVector(c,z,nelg); #endif GenmapScalar den,alpha,beta,rz0,rz1=0,rz2,rr; @@ -54,7 +50,7 @@ int flex_cg(GenmapHandle h,GenmapComm c,mgData d,GenmapVector ri, i=0; while(iGENMAP_TOL){ - GenmapLaplacian(h,c,p,w); + GenmapLaplacian(h,c,p->data,w->data); den=GenmapDotVector(p,w); GenmapGop(c,&den,1,GENMAP_SCALAR,GENMAP_SUM); @@ -71,7 +67,7 @@ int flex_cg(GenmapHandle h,GenmapComm c,mgData d,GenmapVector ri, GenmapCopyVector(z,r); #endif #if ORTH - GenmapOrthogonalizebyOneVector(h,c,z,nelg); + GenmapOrthogonalizebyOneVector(c,z,nelg); #endif rz0=rz1; diff --git a/src/precond/genmap-multigrid-fmg.c b/src/precond/genmap-multigrid-fmg.c index ae6f7431..05b0aa60 100644 --- a/src/precond/genmap-multigrid-fmg.c +++ b/src/precond/genmap-multigrid-fmg.c @@ -4,7 +4,7 @@ #include #include -int inverse(double *x,int level,int iter,GenmapHandle h,GenmapComm c, +int inverse(double *x,int level,int iter,genmap_handle h,GenmapComm c, mgData d) { assert(levelnlevels-2); @@ -22,6 +22,7 @@ int inverse(double *x,int level,int iter,GenmapHandle h,GenmapComm c, j=project_pf_lvl(h,c,d,x,iter,0,level,y); metric_toc(&c->gsc,PROJECTPF); metric_acc(NPROJECTPF,j); + //TODO: 1-orthogonalize // x=y/||y|| @@ -39,7 +40,7 @@ int inverse(double *x,int level,int iter,GenmapHandle h,GenmapComm c, return 0; } -int fmg(GenmapHandle h,GenmapComm c,mgData d,GenmapScalar *z, +int fmg(genmap_handle h,GenmapComm c,mgData d,GenmapScalar *z, int iter,int verbose,GenmapScalar *y) { int nlevels=d->nlevels; diff --git a/src/precond/genmap-multigrid-precon.h b/src/precond/genmap-multigrid-precon.h index c8681e7f..a22b27a4 100644 --- a/src/precond/genmap-multigrid-precon.h +++ b/src/precond/genmap-multigrid-precon.h @@ -38,7 +38,7 @@ struct mgLevel_{ struct mgData_{ struct comm c; - GenmapHandle h; + genmap_handle h; struct gs_data *top; buffer bfr; int nlevels; @@ -75,20 +75,17 @@ void mg_vcycle(GenmapScalar *u,GenmapScalar *rhs,mgData d); void mg_vcycle_lvl(GenmapScalar *u1,GenmapScalar *rhs,mgData d, int lvl_start); -int ortho_one_vector(GenmapHandle h,GenmapComm c,GenmapVector q1, - GenmapLong n); - -int flex_cg(GenmapHandle h,GenmapComm c,mgData d,GenmapVector r, +int flex_cg(genmap_handle h,GenmapComm c,mgData d,GenmapVector r, int maxIter,int verbose,GenmapVector x); -int project_pf(GenmapHandle h,GenmapComm c,mgData d,GenmapVector r, +int project_pf(genmap_handle h,GenmapComm c,mgData d,GenmapVector r, int maxIter,int verbose,GenmapVector x); -int project_pf_lvl(GenmapHandle h,GenmapComm c,mgData d,GenmapScalar *ri, +int project_pf_lvl(genmap_handle h,GenmapComm c,mgData d,GenmapScalar *ri, int maxIter,int verbose,int lvl_start,GenmapScalar *xo); -int rqi(GenmapHandle h,GenmapComm c,mgData d,GenmapVector z, +int rqi(genmap_handle h,GenmapComm c,mgData d,GenmapVector z, int maxIter,int verbose,GenmapVector fiedler); -int fmg(GenmapHandle h,GenmapComm c,mgData d,GenmapScalar *z, +int fmg(genmap_handle h,GenmapComm c,mgData d,GenmapScalar *z, int maxIter,int verbose,GenmapScalar *fiedler); #endif diff --git a/src/precond/genmap-multigrid-projectpf.c b/src/precond/genmap-multigrid-projectpf.c index 023e5263..719d79e9 100644 --- a/src/precond/genmap-multigrid-projectpf.c +++ b/src/precond/genmap-multigrid-projectpf.c @@ -6,7 +6,7 @@ #define MM 505 -int project_pf(GenmapHandle h,GenmapComm c,mgData d,GenmapVector ri, +int project_pf(genmap_handle h,GenmapComm c,mgData d,GenmapVector ri, int maxIter,int verbose,GenmapVector x) { assert(x->size==ri->size); @@ -31,10 +31,7 @@ int project_pf(GenmapHandle h,GenmapComm c,mgData d,GenmapVector ri, for(i=0; idata[i]=0.0,r->data[i]=ri->data[i]; - metric_tic(&d->c,PRECONVCYCLE); - mg_vcycle(z->data,r->data,d); - metric_toc(&d->c,PRECONVCYCLE); - GenmapOrthogonalizebyOneVector(h,c,z,nelg); + GenmapCopyVector(z,r); GenmapCopyVector(p,z); GenmapScalar rz1=GenmapDotVector(r,z); @@ -46,17 +43,25 @@ int project_pf(GenmapHandle h,GenmapComm c,mgData d,GenmapVector ri, GenmapScalar alpha,beta,rz0,rz2,scale; int rank=GenmapCommRank(c); - if(rank==0 && verbose) - printf("projectpf initial rr=%g rz1=%g\n",sqrt(rr),sqrt(rz1)); + if(rank==0 && verbose>1) + printf("\t\tppf initial rr=%g rz1=%g\n",sqrt(rr),sqrt(rz1)); - double tol=sqrt(rr)*1e-7; + double tol=1e-3; i=0; uint j,k; while(igsc,LAPLACIAN); - GenmapLaplacian(h,c,p,w); +#if 0 + GenmapLaplacianWeighted(h,c,p->data,w->data); +#else + GenmapLaplacian(h,c,p->data,w->data); +#endif metric_toc(&c->gsc,LAPLACIAN); + GenmapScalar normw=GenmapDotVector(w,w); + GenmapGop(c,&normw,1,GENMAP_SCALAR,GENMAP_SUM); + normw = sqrt(normw); + GenmapScalar den=GenmapDotVector(p,w); GenmapGop(c,&den,1,GENMAP_SCALAR,GENMAP_SUM); alpha=rz1/den; @@ -80,19 +85,19 @@ int project_pf(GenmapHandle h,GenmapComm c,mgData d,GenmapVector ri, GenmapScalar norm0=GenmapDotVector(z,z); GenmapGop(c,&norm0,1,GENMAP_SCALAR,GENMAP_SUM); - metric_tic(&c->gsc,PRECONVCYCLE); + metric_tic(&c->gsc,VCYCLE); mg_vcycle(z->data,r->data,d); - metric_toc(&c->gsc,PRECONVCYCLE); + metric_toc(&c->gsc,VCYCLE); GenmapScalar norm1=GenmapDotVector(z,z); GenmapGop(c,&norm1,1,GENMAP_SCALAR,GENMAP_SUM); - if(rank==0 && verbose) - printf("vcycle i=%02d norm0=%g norm1=%g\n",i,sqrt(norm0), + if(rank==0 && verbose>2) + printf("\t\tvcycle i=%02d norm0=%g norm1=%g\n",i,sqrt(norm0), sqrt(norm1)); rz0=rz1; - GenmapOrthogonalizebyOneVector(h,c,z,nelg); + GenmapOrthogonalizebyOneVector(c,z,nelg); rz1=GenmapDotVector(r,z); GenmapGop(c,&rz1,1,GENMAP_SCALAR,GENMAP_SUM); @@ -100,15 +105,16 @@ int project_pf(GenmapHandle h,GenmapComm c,mgData d,GenmapVector ri, rz2=GenmapDotVector(r,dz); GenmapGop(c,&rz2,1,GENMAP_SCALAR,GENMAP_SUM); - if(rank==0 && verbose) - printf("projectpf i=%d rr=%g rz1=%g rz2=%g alpha=%g\n", - i,sqrt(rr),sqrt(rz1),sqrt(rz2),alpha); + if(rank==0 && verbose>1) + printf("\t\tppf i=%d norm(w)= %g den=%g rr=%g rz1=%g rz2=%g alpha=%g\n", + i,normw,den,sqrt(rr),sqrt(rz1),sqrt(rz2),alpha); beta=rz2/rz0; GenmapAxpbyVector(p,z,1.0,p,beta); i++; + metric_tic(&c->gsc,PROJECT); for(k=0; kdata[k]-=P[(MM-1)*lelt+k]; + metric_toc(&c->gsc,PROJECT); } GenmapDestroyVector(z),GenmapDestroyVector(w); @@ -134,7 +141,7 @@ int project_pf(GenmapHandle h,GenmapComm c,mgData d,GenmapVector ri, return i; } -int project_pf_lvl(GenmapHandle h,GenmapComm c,mgData d,GenmapScalar *ri, +int project_pf_lvl(genmap_handle h,GenmapComm c,mgData d,GenmapScalar *ri, int maxIter,int verbose,int lvl_start,GenmapScalar *xo) { assert(lvl_startnlevels-1); @@ -160,10 +167,10 @@ int project_pf_lvl(GenmapHandle h,GenmapComm c,mgData d,GenmapScalar *ri, for(i=0; idata[i]=0.0,r->data[i]=ri[i]; - metric_tic(&d->c,PRECONVCYCLE); + metric_tic(&d->c,VCYCLE); mg_vcycle_lvl(z->data,r->data,d,lvl_start); - metric_toc(&d->c,PRECONVCYCLE); - GenmapOrthogonalizebyOneVector(h,c,z,nelg); + metric_toc(&d->c,VCYCLE); + GenmapOrthogonalizebyOneVector(c,z,nelg); GenmapCopyVector(p,z); GenmapScalar rz1=GenmapDotVector(r,z); @@ -182,7 +189,7 @@ int project_pf_lvl(GenmapHandle h,GenmapComm c,mgData d,GenmapScalar *ri, i=0; while(igsc,LAPLACIAN); - //GenmapLaplacian(h,c,p,w); + //GenmapLaplacian(h,c,p->data,w->data); //metric_toc(&c->gsc,LAPLACIAN); metric_tic(&c->gsc,PRECONAX); csr_mat_gather(M,M->gsh,p->data,d->buf,&buf); @@ -212,15 +219,15 @@ int project_pf_lvl(GenmapHandle h,GenmapComm c,mgData d,GenmapScalar *ri, GenmapScalar norm0=GenmapDotVector(z,z); GenmapGop(c,&norm0,1,GENMAP_SCALAR,GENMAP_SUM); - metric_tic(&c->gsc,PRECONVCYCLE); + metric_tic(&c->gsc,VCYCLE); mg_vcycle_lvl(z->data,r->data,d,lvl_start); - metric_toc(&c->gsc,PRECONVCYCLE); + metric_toc(&c->gsc,VCYCLE); GenmapScalar norm1=GenmapDotVector(z,z); GenmapGop(c,&norm1,1,GENMAP_SCALAR,GENMAP_SUM); rz0=rz1; - GenmapOrthogonalizebyOneVector(h,c,z,nelg); + GenmapOrthogonalizebyOneVector(c,z,nelg); rz1=GenmapDotVector(r,z); GenmapGop(c,&rz1,1,GENMAP_SCALAR,GENMAP_SUM); diff --git a/src/precond/genmap-multigrid-rqi.c b/src/precond/genmap-multigrid-rqi.c index 30fde915..1b7f86f8 100644 --- a/src/precond/genmap-multigrid-rqi.c +++ b/src/precond/genmap-multigrid-rqi.c @@ -6,39 +6,118 @@ // Input z should be orthogonal to 1-vector, have unit norm. // RQI should not change z. -int rqi(GenmapHandle h,GenmapComm c,mgData d,GenmapVector z, - int iter,int verbose,GenmapVector y) +int rqi(genmap_handle h,GenmapComm c,mgData d,GenmapVector z,int max_iter,int verbose,GenmapVector y) { assert(z->size==y->size); - int ppfi; - metric_tic(&c->gsc,PROJECTPF); - ppfi=project_pf(h,c,d,z,20,verbose,y); - metric_toc(&c->gsc,PROJECTPF); - metric_acc(NPROJECTPF,ppfi); + uint lelt=z->size; + GenmapVector err; GenmapCreateVector(&err,lelt); - GenmapVector err; GenmapCreateVector(&err,z->size); - GenmapLong nelg=GenmapGetNGlobalElements(h); + struct comm *gsc=&c->gsc; int rank=GenmapCommRank(GenmapGetGlobalComm(h)); - uint i; - for(i=0; igsc,PROJECTPF); - ppfi=project_pf(h,c,d,z,100,verbose,y); - metric_toc(&c->gsc,PROJECTPF); - metric_acc(NPROJECTPF,ppfi); - GenmapOrthogonalizebyOneVector(h,c,y,nelg); + uint i,j,k,l; + for(i=0; i<20; i++){ + GenmapScalar norm=GenmapDotVector(y,y); + GenmapGop(c,&norm,1,GENMAP_SCALAR,GENMAP_SUM); + GenmapScalar normi=1.0/sqrt(norm); + + if(rank==0 && verbose>0) + printf("\ti=%02d #ppf=%02d |y|=%g\n",i,ppfi,sqrt(norm)); + + GenmapAxpbyVector(z,z,0.0,y,normi); + GenmapOrthogonalizebyOneVector(c,z,nelg); + +#if defined(GENMAP_GRAMMIAN) + metric_tic(gsc,GRAMMIAN); + int N = i + 1; + //if k>1; + // Z(:,k)=z-Z(:,1:k-1)*(Z(:,1:k-1)'*z); + // Z(:,k)=Z(:,k)/norm(Z(:,k)); + //end; + if(i>0){ + // rhs = Z[1:k-1,:]*z + for(j=0; jdata[l]; + } + // Global reduction rhs[j] + comm_allreduce(gsc,gs_double,gs_add,rhs,i,buf); + + //Z[k,:] = z[:] - Z[:,1:lelt]*rhs[:] + for(l=0; ldata[l]; + + for(j=0; j2); + + for(j=0; jdata[j]=0.0; + + for(j=0; jdata[k]+=Z[j*lelt+k]*v[j]; + } + GenmapOrthogonalizebyOneVector(c,z,nelg); + }else{ + //Z(k,:) = z; + for(l=0; ldata[l]; + } + metric_toc(gsc,GRAMMIAN); +#endif + + metric_tic(gsc,PROJECTPF); + ppfi=project_pf(h,c,d,z,20,verbose,y); + metric_toc(gsc,PROJECTPF); + metric_acc(NPROJECTPF,ppfi); + GenmapOrthogonalizebyOneVector(c,y,nelg); GenmapScalar lambda=GenmapDotVector(y,z); GenmapGop(c,&lambda,1,GENMAP_SCALAR,GENMAP_SUM); @@ -51,9 +130,15 @@ int rqi(GenmapHandle h,GenmapComm c,mgData d,GenmapVector z, GenmapScalar norm1=GenmapDotVector(y,y); GenmapGop(c,&norm1,1,GENMAP_SCALAR,GENMAP_SUM); GenmapScalar normi1=1.0/sqrt(norm1); - metric_acc(END+i,norme*normi1); } + GenmapFree(Z); + GenmapFree(GZ); + GenmapFree(M); + GenmapFree(rhs); + GenmapFree(v); + GenmapFree(buf); + GenmapDestroyVector(err); return i; diff --git a/src/precond/genmap-multigrid-vcycle.c b/src/precond/genmap-multigrid-vcycle.c index c201b126..f8b42252 100644 --- a/src/precond/genmap-multigrid-vcycle.c +++ b/src/precond/genmap-multigrid-vcycle.c @@ -22,7 +22,7 @@ void mg_vcycle(GenmapScalar *u1,GenmapScalar *rhs,mgData d) for(i=0; ih; + genmap_handle h=d->h; for(lvl=0; lvla,s->unit_size,proc,sizeof(uint),&cr); crystal_free(&cr); - metric_toc(c,RCBTRANSFER); GenmapFree(proc); diff --git a/src/sort/hypercube.c b/src/sort/hypercube.c index 5507cb77..5513b0ec 100644 --- a/src/sort/hypercube.c +++ b/src/sort/hypercube.c @@ -138,9 +138,7 @@ int parallel_hypercube_sort(struct hypercube *data,struct comm *c) update_probe_counts(data,c); metric_toc(c,UPDATEPROBE); } - metric_tic(c,RCBTRANSFER); transfer_elem(data,c); - metric_toc(c,RCBTRANSFER); // split the communicator struct comm nc; diff --git a/src/sort/sort-impl.h b/src/sort/sort-impl.h index 34538d54..2ab2ca2c 100644 --- a/src/sort/sort-impl.h +++ b/src/sort/sort-impl.h @@ -12,8 +12,7 @@ double get_scalar(struct array *a,uint i,uint off,uint usize,gs_dom type); void get_extrema(void *extrema,struct sort *s,uint field,struct comm *c); int set_dest(uint *proc,uint np,ulong start,uint size,ulong nelem); -int load_balance(struct array *a,size_t size,struct comm *c, - struct crystal *cr); +int load_balance(struct array *a,size_t size,struct comm *c,struct crystal *cr); int sort_local(struct sort *s); #endif diff --git a/src/sort/sort.c b/src/sort/sort.c index 46831a29..fd7546aa 100644 --- a/src/sort/sort.c +++ b/src/sort/sort.c @@ -32,8 +32,7 @@ int set_dest(uint *proc,uint np,ulong start,uint size,ulong nelem) return 0; } -int load_balance(struct array *a,size_t size,struct comm *c, - struct crystal *cr) +int load_balance(struct array *a,size_t size,struct comm *c,struct crystal *cr) { slong in=a->n,out[2][1],buf[2][1]; comm_scan(out,c,gs_long,gs_add,&in,1,buf); @@ -121,18 +120,12 @@ int parallel_sort_private(struct sort *data,struct comm *c){ break; } - metric_tic(c,LOADBALANCE0); if(balance){ - metric_tic(c,LOADBALANCE1); struct crystal cr; crystal_init(&cr,c); load_balance(a,usize,c,&cr); - metric_toc(c,LOADBALANCE1); - metric_tic(c,LOCALSORT); sort_local(data); - metric_toc(c,LOCALSORT); crystal_free(&cr); } - metric_toc(c,LOADBALANCE0); comm_free(&dup); diff --git a/tests/t200-laplacian.c b/tests/t200-laplacian.c index 3ff0a1a8..057afad3 100644 --- a/tests/t200-laplacian.c +++ b/tests/t200-laplacian.c @@ -22,7 +22,7 @@ int main(int argc,char *argv[]){ Mesh mesh; read_co2_mesh(&mesh,argv[1],&comm); - GenmapHandle gh; GenmapInit(&gh,MPI_COMM_WORLD); + genmap_handle gh; genmap_init(&gh,MPI_COMM_WORLD); GenmapSetNLocalElements(gh,mesh->nelt); GenmapSetNVertices(gh,mesh->nVertex); @@ -52,8 +52,8 @@ int main(int argc,char *argv[]){ GenmapDestroyVector(v); GenmapDestroyVector(u); - MeshFree(mesh); - GenmapFinalize(gh); + mesh_free(mesh); + genmap_finalize(gh); comm_free(&comm); MPI_Finalize(); diff --git a/tests/t210-levels.c b/tests/t210-levels.c index 32d78b47..c98025e8 100644 --- a/tests/t210-levels.c +++ b/tests/t210-levels.c @@ -22,7 +22,7 @@ int main(int argc,char *argv[]){ Mesh mesh; read_co2_mesh(&mesh,argv[1],&comm); - GenmapHandle gh; GenmapInit(&gh,MPI_COMM_WORLD); + genmap_handle gh; genmap_init(&gh,MPI_COMM_WORLD); GenmapSetNLocalElements(gh,mesh->nelt); GenmapSetNVertices(gh,mesh->nVertex); @@ -61,8 +61,8 @@ int main(int argc,char *argv[]){ mgFree(d); - GenmapFinalize(gh); - MeshFree(mesh); + genmap_finalize(gh); + mesh_free(mesh); comm_free(&comm); MPI_Finalize(); diff --git a/tests/t220-interpolation.c b/tests/t220-interpolation.c index 5727de9c..72db70ca 100644 --- a/tests/t220-interpolation.c +++ b/tests/t220-interpolation.c @@ -21,7 +21,7 @@ int main(int argc,char *argv[]){ Mesh mesh; read_co2_mesh(&mesh,argv[1],&comm); - GenmapHandle gh; GenmapInit(&gh,MPI_COMM_WORLD); + genmap_handle gh; genmap_init(&gh,MPI_COMM_WORLD); GenmapSetNLocalElements(gh,mesh->nelt); GenmapSetNVertices(gh,mesh->nVertex); @@ -73,8 +73,8 @@ int main(int argc,char *argv[]){ buffer_free(&buf); mgFree(d); - GenmapFinalize(gh); - MeshFree(mesh); + genmap_finalize(gh); + mesh_free(mesh); comm_free(&comm); MPI_Finalize(); diff --git a/tests/t230-flex-cg.c b/tests/t230-flex-cg.c index 2aec221a..ae1564bc 100644 --- a/tests/t230-flex-cg.c +++ b/tests/t230-flex-cg.c @@ -23,8 +23,8 @@ int main(int argc,char *argv[]){ } Mesh mesh; - read_re2_mesh(&mesh,argv[1],&comm); - read_co2_file( mesh,argv[2],&comm); + read_geometry(&mesh,argv[1],&comm); + read_connectivity( mesh,argv[2],&comm); GenmapInt i,j; @@ -71,8 +71,8 @@ int main(int argc,char *argv[]){ } if(rcb_l){ - struct array a; array_init(elm_rcb,&a,mesh->nelt); a.n=mesh->nelt; - elm_rcb *ptr=a.ptr; + struct array a; array_init(struct rcb_element,&a,mesh->nelt); a.n=mesh->nelt; + struct rcb_element *ptr=a.ptr; for(i=0; inelt; i++){ ptr[i].orig=i; ptr[i].coord[0]=0.0; @@ -97,7 +97,7 @@ int main(int argc,char *argv[]){ for(i=0; inelt; i++) ptr[i].proc=i; - sarray_sort(elm_rcb,a.ptr,a.n,orig,0,&buf); + sarray_sort(struct rcb_element,a.ptr,a.n,orig,0,&buf); ptr=a.ptr; Point pp=mesh->elements.ptr; @@ -119,7 +119,7 @@ int main(int argc,char *argv[]){ free(coords); buffer_free(&buf); - GenmapHandle gh; GenmapInit(&gh,MPI_COMM_WORLD); + genmap_handle gh; genmap_init(&gh,MPI_COMM_WORLD); GenmapSetNLocalElements(gh,mesh->nelt); GenmapSetNVertices(gh,mesh->nVertex); @@ -169,8 +169,8 @@ int main(int argc,char *argv[]){ mgFree(d); - GenmapFinalize(gh); - MeshFree(mesh); + genmap_finalize(gh); + mesh_free(mesh); comm_free(&comm); MPI_Finalize(); diff --git a/tests/t235-project-pf.c b/tests/t235-project-pf.c index 2a3ea8b3..d56c713a 100644 --- a/tests/t235-project-pf.c +++ b/tests/t235-project-pf.c @@ -23,8 +23,8 @@ int main(int argc,char *argv[]){ } Mesh mesh; - read_re2_mesh(&mesh,argv[1],&comm); - read_co2_file( mesh,argv[2],&comm); + read_geometry(&mesh,argv[1],&comm); + read_connectivity( mesh,argv[2],&comm); GenmapInt i,j; @@ -71,8 +71,8 @@ int main(int argc,char *argv[]){ } if(rcb_l){ - struct array a; array_init(elm_rcb,&a,mesh->nelt); a.n=mesh->nelt; - elm_rcb *ptr=a.ptr; + struct array a; array_init(struct rcb_element,&a,mesh->nelt); a.n=mesh->nelt; + struct rcb_element *ptr=a.ptr; for(i=0; inelt; i++){ ptr[i].orig=i; ptr[i].coord[0]=0.0; @@ -97,7 +97,7 @@ int main(int argc,char *argv[]){ for(i=0; inelt; i++) ptr[i].proc=i; - sarray_sort(elm_rcb,a.ptr,a.n,orig,0,&buf); + sarray_sort(struct rcb_element,a.ptr,a.n,orig,0,&buf); ptr=a.ptr; Point pp=mesh->elements.ptr; @@ -119,7 +119,7 @@ int main(int argc,char *argv[]){ free(coords); buffer_free(&buf); - GenmapHandle gh; GenmapInit(&gh,MPI_COMM_WORLD); + genmap_handle gh; genmap_init(&gh,MPI_COMM_WORLD); GenmapSetNLocalElements(gh,mesh->nelt); GenmapSetNVertices(gh,mesh->nVertex); @@ -169,8 +169,8 @@ int main(int argc,char *argv[]){ mgFree(d); - GenmapFinalize(gh); - MeshFree(mesh); + genmap_finalize(gh); + mesh_free(mesh); comm_free(&comm); MPI_Finalize(); diff --git a/tests/t240-rqi.c b/tests/t240-rqi.c index 8732132a..f0b185c6 100644 --- a/tests/t240-rqi.c +++ b/tests/t240-rqi.c @@ -23,8 +23,8 @@ int main(int argc,char *argv[]){ } Mesh mesh; - read_re2_mesh(&mesh,argv[1],&comm); - read_co2_file( mesh,argv[2],&comm); + read_geometry(&mesh,argv[1],&comm); + read_connectivity( mesh,argv[2],&comm); GenmapInt i,j; @@ -71,8 +71,8 @@ int main(int argc,char *argv[]){ } if(rcb_l){ - struct array a; array_init(elm_rcb,&a,mesh->nelt); a.n=mesh->nelt; - elm_rcb *ptr=a.ptr; + struct array a; array_init(struct rcb_element,&a,mesh->nelt); a.n=mesh->nelt; + struct rcb_element *ptr=a.ptr; for(i=0; inelt; i++){ ptr[i].orig=i; ptr[i].coord[0]=0.0; @@ -97,7 +97,7 @@ int main(int argc,char *argv[]){ for(i=0; inelt; i++) ptr[i].proc=i; - sarray_sort(elm_rcb,a.ptr,a.n,orig,0,&buf); + sarray_sort(struct rcb_element,a.ptr,a.n,orig,0,&buf); ptr=a.ptr; Point pp=mesh->elements.ptr; @@ -119,7 +119,7 @@ int main(int argc,char *argv[]){ free(coords); buffer_free(&buf); - GenmapHandle gh; GenmapInit(&gh,MPI_COMM_WORLD); + genmap_handle gh; genmap_init(&gh,MPI_COMM_WORLD); GenmapSetNLocalElements(gh,mesh->nelt); GenmapSetNVertices(gh,mesh->nVertex); @@ -161,8 +161,8 @@ int main(int argc,char *argv[]){ GenmapDestroyVector(x); GenmapDestroyVector(r); - GenmapFinalize(gh); - MeshFree(mesh); + genmap_finalize(gh); + mesh_free(mesh); comm_free(&comm); MPI_Finalize();