Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

HDF5 output capability #7

Open
wants to merge 7 commits into
base: main
Choose a base branch
from
Open

HDF5 output capability #7

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
acac44a
Initial commit of HDF5 output capability
emprice Jan 27, 2024
0a3178d
Bugfix: Indexing error when assigning hyperslabs
emprice Jan 30, 2024
f0e108e
Bugfix: typo in conditionally-compiled section
emprice Jan 31, 2024
50e7c12
Bugfix: All ranks must select the same HDF5 chunk size
emprice Feb 1, 2024
673f2bf
Fixing bug when writing ghost cells to file
emprice Jul 3, 2024
a6755d1
Changing HDF5 dependency from .par to .opt option
emprice Jul 5, 2024
245e829
Merge branch 'FARGO3D:main' into feature/hdf5-output
emprice Jul 5, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
104 changes: 53 additions & 51 deletions src/LowTasks.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ int PrimitiveVariables () {
var |= VY;
#endif
#ifdef Z
var |= VZ;
var |= VZ;
#endif

#ifdef MHD
Expand Down Expand Up @@ -116,7 +116,7 @@ void MakeDir (char *string) {
/* If all processes see the same partition, only the first process
will create the directory. Alternatively, they will create as
many directories as necessary. For instance, if we have say 4 PEs per node
and each node sees its own scratchdir, nbprocesses/4
and each node sees its own scratchdir, nbprocesses/4
mkdir() commands will be issued */
if (CPU_Rank) MPI_Recv (&foo, 1, MPI_INT, CPU_Rank-1, 53, MPI_COMM_WORLD, &fargostat);
dir = opendir (string);
Expand Down Expand Up @@ -199,7 +199,7 @@ void InitSpace() {
real dx,dy, dz;
real x0;
int i,j,k;

FILE *domain;
char domain_out[512];
real ymin, zmin, xmin;
Expand All @@ -212,7 +212,7 @@ void InitSpace() {
int init = 0;
int j_global, j_local;


if (*SPACING=='F') { //Fixed spacing
#ifdef X
masterprint("Warning: zone spacing will be taken from the files domain_i.dat.\n");
Expand All @@ -232,7 +232,7 @@ void InitSpace() {
already_x = YES;
}
fclose(domain);
#endif
#endif
#ifdef Y
sprintf(domain_out, "%s%s", OUTPUTDIR, "domain_y.dat");
domain = fopen(domain_out, "r");
Expand All @@ -250,7 +250,7 @@ void InitSpace() {
}
already_y = YES;
}

fclose(domain);
#endif
#ifdef Z
Expand Down Expand Up @@ -297,11 +297,11 @@ void InitSpace() {
}
// Fill ghost zones using the fact that du is constant and we bisect in reverse order
if (J == 0) {
for (j = 0; j < NGHY; j++)
for (j = 0; j < NGHY; j++)
Ymin(NGHY - (j+1)) = bisect(0.5*YMIN, Ymin(NGHY-j), NY+1, uy, 1);
}
if (J == Ncpu_x - 1) {
for (j = 0; j < NGHY; j++)
for (j = 0; j < NGHY; j++)
Ymin(Ny+NGHY+j+1) = bisect(Ymin(Ny+NGHY+j), 1.5*YMAX, NY+1, uy, 0);
}

Expand All @@ -314,7 +314,7 @@ void InitSpace() {
}
#endif //Z
}


else {

Expand All @@ -333,7 +333,7 @@ void InitSpace() {
#else
dz = 0;
#endif

if (((toupper(*SPACING)) == 'L') && ((toupper(*(SPACING+1))) == 'O')) { //Logarithmic
masterprint("Warning: The Y spacing is logarithmic.\n");
dy = (log(YMAX)-log(YMIN))/NY;
Expand Down Expand Up @@ -363,7 +363,7 @@ void InitSpace() {
Xmin(i) = XMIN + dx*(i-NGHX);
#else
Xmin(i) = 0.0;
#endif
#endif
}
}

Expand All @@ -381,10 +381,11 @@ void InitSpace() {
InvDiffXmed(i) = 1./(Xmed(i)-Xmed(i-1));
}
InvDiffXmed(0) = 1./( Xmed(0)- (Xmed(Nx-1)-(XMAX-XMIN) ));


MPI_Barrier(MPI_COMM_WORLD);


#ifndef HDF5
if (!already_x) {
if(CPU_Master) {
sprintf(domain_out, "%s%s", OUTPUTDIR, "domain_x.dat");
Expand All @@ -395,17 +396,17 @@ void InitSpace() {
}
}
}

if (!already_y) {
if (CPU_Rank > 0) { // Force sequential write
MPI_Recv (&relay, 1, MPI_INT, CPU_Rank-1, 42, MPI_COMM_WORLD, MPI_STATUS_IGNORE);
}

sprintf(domain_out, "%s%s", OUTPUTDIR, "domain_y.dat");
if(CPU_Master) {
domain = fopen(domain_out, "w");
jmin = 0;
jmax = Ny+NGHY+1;
jmax = Ny+NGHY+1;
}
else {
if (CPU_Rank < Ncpu_x) {
Expand All @@ -428,17 +429,17 @@ void InitSpace() {
}

MPI_Barrier (MPI_COMM_WORLD);

if (!already_z) {
if (CPU_Rank > 0) { // Force sequential read
if (CPU_Rank > 0) { // Force sequential write
MPI_Recv (&relay, 1, MPI_INT, CPU_Rank-1, 43, MPI_COMM_WORLD, MPI_STATUS_IGNORE);
}
sprintf(domain_out, "%s%s", OUTPUTDIR, "domain_z.dat");
if(CPU_Master) {
domain = fopen(domain_out, "w");
jmin = 0;
jmax = Nz+NGHZ+1;
}
}
else {
if (J == 0) {
domain = fopen(domain_out, "a");
Expand All @@ -454,12 +455,13 @@ void InitSpace() {
}
fclose(domain);
}
if (CPU_Rank < CPU_Number-1) { // Force sequential read
if (CPU_Rank < CPU_Number-1) { // Force sequential write
MPI_Send (&relay, 1, MPI_INT, CPU_Rank+1, 43, MPI_COMM_WORLD);
}

MPI_Barrier (MPI_COMM_WORLD);
}
#endif
}


Expand Down Expand Up @@ -494,7 +496,7 @@ for(i=1;i<Nx+2*NGHX; i++) {
Szj(j) = 1.0;
#endif
InvVj(j) = 1./(Sxj(j));
}
}
for (k = 0; k<Nz+2*NGHZ; k++) {
#ifdef Z
Sxk(k) = Zmin(k+1)-Zmin(k);
Expand All @@ -503,7 +505,7 @@ for(i=1;i<Nx+2*NGHX; i++) {
#endif
#ifdef Z
Syk(k) = Zmin(k+1)-Zmin(k);
#else
#else
Syk(k) = 1.0;
#endif
Szk(k) = 1.0;
Expand Down Expand Up @@ -541,8 +543,8 @@ for(i=1;i<Nx+2*NGHX; i++) {
-Ymin(j)*Ymin(j));
Syj(j) = Ymin(j)*Ymin(j);
Szj(j) = 0.5*(Ymin(j+1)*Ymin(j+1)
-Ymin(j)*Ymin(j));
InvVj(j) = 3./((Ymin(j+1)*Ymin(j+1)*Ymin(j+1) -
-Ymin(j)*Ymin(j));
InvVj(j) = 3./((Ymin(j+1)*Ymin(j+1)*Ymin(j+1) -
Ymin(j)*Ymin(j)*Ymin(j)));
}
for (k = 0; k<Nz+2*NGHZ; k++) {
Expand All @@ -553,7 +555,7 @@ for(i=1;i<Nx+2*NGHX; i++) {
#else
Sxk(k) = 1.0;
Syk(k) = 1.0;
Szk(k) = 1.0;
Szk(k) = 1.0;
#endif
}
#endif
Expand Down Expand Up @@ -597,10 +599,10 @@ void CreateFields() {
Reduction2D = CreateField2D ("Reduction2D", YZ);

#if (defined(X) || defined(MHD))

Mpx = CreateField ("Moment_Plus_X" , 0, 1,0,0);
Mmx = CreateField ("Moment_Minus_X", 0, 1,0,0);

Vxhy = CreateField2D ("Vxhy" , YZ);
Vxhyr = CreateField2D ("Vxhyr" , YZ);
Vxhz = CreateField2D ("Vxhz" , YZ);
Expand All @@ -615,7 +617,7 @@ void CreateFields() {
Mpy = CreateField("Moment_Plus_Y" , 0,0,1,0);
Mmy = CreateField("Moment_Minus_Y", 0,0,1,0);
#endif

#if (defined(Z) || defined(MHD))
Mpz = CreateField("Moment_Plus_Z" , 0,0,0,1);
Mmz = CreateField("Moment_Minus_Z", 0,0,0,1);
Expand All @@ -627,16 +629,16 @@ void CreateFields() {
// be aliased wherever
// reductions are
// needed

DensStar = CreateField("DensStar" , 0,0,0,0);
Qs = CreateField("Qs" , 0,0,0,0);
Pressure = CreateField("Pressure" , 0,0,0,0);
Total_Density = CreateField("Total_Density", 0,0,0,0);

QL = CREATEFIELDALIAS("QLeft", Pressure, 0);
QR = CreateField("QRight", 0,0,0,0);

#ifdef RAM
#ifdef RAM
PhiStarmin = CreateField("PhiStarmin", 0,0,0,0);
UStarmin = CreateField("UStarmin", 0,0,0,0);
#endif
Expand All @@ -654,7 +656,7 @@ void CreateFields() {
Sdiffyfzf = CREATEFIELDALIAS("Sdiffyfzf",Mpy,0);
#endif
#endif

#ifdef MHD
Bx = CreateField("bx", BX,1,0,0);
By = CreateField("by", BY,0,1,0);
Expand Down Expand Up @@ -694,13 +696,13 @@ void CreateFields() {
#ifdef AMBIPOLARDIFFUSION
EtaAD = CreateField("EtaAD",0,0,0,0);
#endif

//Claim ownership of storage area
*(Emfx->owner) = Emfx;
*(Emfy->owner) = Emfy;
*(Emfz->owner) = Emfz;
Divergence = CreateField("divb", 0, 0,0,0);

Divergence = CreateField("divb", 0, 0,0,0);

#endif

Expand All @@ -712,7 +714,7 @@ real ComputeMass() {
real totalmass;

real *rho;

INPUT (Density);

rho = Density->field_cpu;
Expand Down Expand Up @@ -823,7 +825,7 @@ int RestartSimulation(int n) {
__Restart = RestartVTK;
else
__Restart = RestartDat;

if (Dat2vtk) {
Merge = YES;
__Restart = RestartDat;
Expand All @@ -849,11 +851,11 @@ int RestartSimulation(int n) {
__Restart(Bz, n);
#endif
#endif

#ifdef MPIIO
MPI_Offset offset;
offset = 0; //We start at the begining of the file

offset = ParallelIO(Density, n, MPI_MODE_RDONLY, offset,FALSE);
if(Fluidtype != DUST) offset = ParallelIO(Energy, n, MPI_MODE_RDONLY, offset,FALSE);
#ifdef X
Expand All @@ -869,11 +871,11 @@ int RestartSimulation(int n) {
if(Fluidtype == GAS){
offset = ParallelIO(Bx, n, MPI_MODE_RDONLY, offset,FALSE);
offset = ParallelIO(By, n, MPI_MODE_RDONLY, offset,FALSE);
offset = ParallelIO(Bz, n, MPI_MODE_RDONLY, offset,FALSE);
offset = ParallelIO(Bz, n, MPI_MODE_RDONLY, offset,FALSE);
}
#endif
#endif

begin = n*NINTERM;
if (PostRestart)
PostRestartHook ();
Expand Down Expand Up @@ -903,17 +905,17 @@ void RestartVTK(Field *f, int n) {
masterprint("Error reading %s\n", filename);
exit(1);
}

masterprint("Reading %s\n", filename);

while(1) {
temp = fscanf(fi, "%s\n", line);
if (strcmp(line,"LOOKUP_TABLE") == 0){
temp = fscanf(fi, "%s\n", line);
break;
}
}

i = j = k = 0;

#ifndef SPHERICAL
Expand Down Expand Up @@ -945,9 +947,9 @@ void RestartVTK(Field *f, int n) {
masterprint("Error reading %s\n", filename);
exit(1);
}

masterprint("Reading %s\n", filename);

while(1) {
temp = fscanf(fi, "%s\n", line);
if (strcmp(line,"LOOKUP_TABLE") == 0){
Expand All @@ -956,7 +958,7 @@ void RestartVTK(Field *f, int n) {
break;
}
}

i = j = k = 0;

origin = Y0+Z0*NX*NY;
Expand Down Expand Up @@ -1009,7 +1011,7 @@ void RestartDat(Field *field, int n) {
exit(1);
}
masterprint("Reading %s\n", filename);

for (k=NGHZ; k<Nz+NGHZ; k++) {
for (j=NGHY; j<Ny+NGHY; j++) {
temp = fread(f+j*(Nx+2*NGHX)+k*Stride+NGHX, sizeof(real), Nx, fi);
Expand All @@ -1022,7 +1024,7 @@ void RestartDat(Field *field, int n) {
MPI_Finalize();
}
}

MPI_Barrier(MPI_COMM_WORLD);
if(Restart_Full == YES) {
sprintf(filename, "%s%s%d.dat", OUTPUTDIR, name, n);
Expand All @@ -1032,7 +1034,7 @@ void RestartDat(Field *field, int n) {
exit(1);
}
masterprint("Reading %s\n", filename);

origin = (z0cell)*NX*NY + (y0cell)*NX; //z0cell and y0cell are global variables.
for (k=NGHZ; k<Nz+NGHZ; k++) {
fseek(fi, (origin+(k-NGHZ)*NX*NY)*sizeof(real), SEEK_SET); // critical part
Expand Down
Loading